Browse Source

[DSIP-46] Unify the logic task params data structure between front-end and back-end (#16193)

dev
Wenjun Ruan 5 months ago committed by GitHub
parent
commit
ec6a3e0727
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 20
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
  2. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  3. 24
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  4. 20
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
  5. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java
  6. 30
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
  7. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
  8. 3
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
  9. 22
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java
  10. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
  11. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
  12. 9
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java
  13. 9
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java
  14. 6
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
  15. 133
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java
  16. 2
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
  17. 33
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java
  18. 4
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java
  19. 21
      dolphinscheduler-dao/src/test/resources/logback.xml
  20. 2
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
  21. 59
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java
  22. 15
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java
  23. 13
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
  24. 77
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
  25. 7
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java
  26. 11
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java
  27. 15
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java
  28. 139
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java
  29. 51
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java
  30. 51
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java
  31. 21
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java
  32. 15
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java
  33. 20
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java
  34. 9
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java
  35. 13
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java
  36. 13
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskPluginFactory.java
  37. 122
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java
  38. 10
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java
  39. 4
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
  40. 48
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/TaskUtils.java
  41. 1
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
  42. 47
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactoryTest.java
  43. 10
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java
  44. 8
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
  45. 39
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/TaskUtilsTest.java
  46. 129
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java
  47. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  48. 29
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
  49. 155
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java
  50. 457
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java
  51. 7
      dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
  52. 5
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
  53. 21
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ILogicTaskChannel.java
  54. 10
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannel.java
  55. 13
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java
  56. 21
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
  57. 96
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java
  58. 4
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DynamicInputParameter.java
  59. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java
  60. 51
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java
  61. 41
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java
  62. 39
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java
  63. 6
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DynamicParameters.java
  64. 116
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java
  65. 5
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java
  66. 79
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java
  67. 5
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java
  68. 48
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/AbstractLogicTaskChannel.java
  69. 31
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannel.java
  70. 34
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannelFactory.java
  71. 30
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannel.java
  72. 38
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannelFactory.java
  73. 30
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannel.java
  74. 38
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannelFactory.java
  75. 30
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannel.java
  76. 39
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannelFactory.java
  77. 30
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannel.java
  78. 39
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannelFactory.java
  79. 55
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskTypeUtils.java
  80. 50
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java
  81. 44
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManagerTest.java
  82. 18
      dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannel.java
  83. 12
      dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannelFactory.java
  84. 14
      dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannel.java
  85. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannelFactory.java
  86. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java
  87. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannelFactory.java
  88. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java
  89. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java
  90. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java
  91. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java
  92. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java
  93. 9
      dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java
  94. 16
      dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannel.java
  95. 8
      dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java
  96. 16
      dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java
  97. 23
      dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java
  98. 15
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java
  99. 9
      dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java
  100. 26
      dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java
  101. Some files were not shown because too many files have changed in this diff Show More

20
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java

@ -19,22 +19,14 @@ package org.apache.dolphinscheduler.api;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.common.CommonConfiguration; import org.apache.dolphinscheduler.common.CommonConfiguration;
import org.apache.dolphinscheduler.common.enums.PluginType;
import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.DaoConfiguration;
import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider;
import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
import org.apache.dolphinscheduler.service.ServiceConfiguration; import org.apache.dolphinscheduler.service.ServiceConfiguration;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -68,17 +60,7 @@ public class ApiApplicationServer {
@EventListener @EventListener
public void run(ApplicationReadyEvent readyEvent) { public void run(ApplicationReadyEvent readyEvent) {
log.info("Received spring application context ready event will load taskPlugin and write to DB"); log.info("Received spring application context ready event will load taskPlugin and write to DB");
// install task plugin
TaskPluginManager.loadPlugin();
DataSourceProcessorProvider.initialize(); DataSourceProcessorProvider.initialize();
for (Map.Entry<String, TaskChannelFactory> entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) { TaskPluginManager.loadTaskPlugin();
String taskPluginName = entry.getKey();
TaskChannelFactory taskChannelFactory = entry.getValue();
List<PluginParams> params = taskChannelFactory.getParams();
String paramsJson = PluginParamsTransfer.transferParamsToJson(params);
PluginDefine pluginDefine = new PluginDefine(taskPluginName, PluginType.TASK.getDesc(), paramsJson);
pluginDao.addOrUpdatePluginDefine(pluginDefine);
}
} }
} }

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@ -89,8 +89,8 @@ import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest; import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.cron.CronUtils;
@ -361,7 +361,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
// find out the process definition code // find out the process definition code
Set<Long> processDefinitionCodeSet = new HashSet<>(); Set<Long> processDefinitionCodeSet = new HashSet<>();
taskDefinitions.stream() taskDefinitions.stream()
.filter(task -> TaskConstants.TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(task.getTaskType())).forEach( .filter(task -> TaskTypeUtils.isSubWorkflowTask(task.getTaskType())).forEach(
taskDefinition -> processDefinitionCodeSet.add(Long.valueOf( taskDefinition -> processDefinitionCodeSet.add(Long.valueOf(
JSONUtils.getNodeString(taskDefinition.getTaskParams(), JSONUtils.getNodeString(taskDefinition.getTaskParams(),
CMD_PARAM_SUB_PROCESS_DEFINE_CODE)))); CMD_PARAM_SUB_PROCESS_DEFINE_CODE))));

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -39,7 +39,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.IMPORT_SUFF
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS; import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOCAL_PARAMS_LIST; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOCAL_PARAMS_LIST;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL; import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.dto.DagDataSchedule; import org.apache.dolphinscheduler.api.dto.DagDataSchedule;
import org.apache.dolphinscheduler.api.dto.treeview.Instance; import org.apache.dolphinscheduler.api.dto.treeview.Instance;
@ -109,12 +109,12 @@ import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionLogDao; import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao;
import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils; import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType; import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.plugin.task.sql.SqlTaskChannelFactory;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager; import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
@ -421,11 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson); throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson);
} }
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
.taskType(taskDefinitionLog.getTaskType())
.taskParams(taskDefinitionLog.getTaskParams())
.dependence(taskDefinitionLog.getDependence())
.build())) {
log.error( log.error(
"Generate task definition list failed, the given task definition parameter is invalided, taskName: {}, taskDefinition: {}", "Generate task definition list failed, the given task definition parameter is invalided, taskName: {}, taskDefinition: {}",
taskDefinitionLog.getName(), taskDefinitionLog); taskDefinitionLog.getName(), taskDefinitionLog);
@ -1386,7 +1382,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
sqlParameters.setLocalParams(Collections.emptyList()); sqlParameters.setLocalParams(Collections.emptyList());
taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters)); taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters));
taskDefinition.setCode(CodeGenerateUtils.genCode()); taskDefinition.setCode(CodeGenerateUtils.genCode());
taskDefinition.setTaskType(TASK_TYPE_SQL); taskDefinition.setTaskType(SqlTaskChannelFactory.NAME);
taskDefinition.setFailRetryTimes(0); taskDefinition.setFailRetryTimes(0);
taskDefinition.setFailRetryInterval(0); taskDefinition.setFailRetryInterval(0);
taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE); taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE);
@ -1615,13 +1611,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
// check whether the process definition json is normal // check whether the process definition json is normal
for (TaskNode taskNode : taskNodes) { for (TaskNode taskNode : taskNodes) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskNode.getType(), taskNode.getParams())) {
.taskType(taskNode.getType())
.taskParams(taskNode.getTaskParams())
.dependence(taskNode.getDependence())
.switchResult(taskNode.getSwitchResult())
.build())) {
log.error("Task node {} parameter invalid.", taskNode.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName());
return result; return result;
} }
@ -1891,7 +1881,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
long subProcessCode = 0L; long subProcessCode = 0L;
// if process is sub process, the return sub id, or sub id=0 // if process is sub process, the return sub id, or sub id=0
if (taskInstance.isSubProcess()) { if (TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) {
TaskDefinition taskDefinition = taskDefinitionMap.get(taskInstance.getTaskCode()); TaskDefinition taskDefinition = taskDefinitionMap.get(taskInstance.getTaskCode());
subProcessCode = Long.parseLong(JSONUtils.parseObject( subProcessCode = Long.parseLong(JSONUtils.parseObject(
taskDefinition.getTaskParams()).path(CMD_PARAM_SUB_PROCESS_DEFINE_CODE).asText()); taskDefinition.getTaskParams()).path(CMD_PARAM_SUB_PROCESS_DEFINE_CODE).asText());

20
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@ -26,8 +26,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.GLOBAL_PARA
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS; import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.common.constants.Constants.PROCESS_INSTANCE_STATE; import static org.apache.dolphinscheduler.common.constants.Constants.PROCESS_INSTANCE_STATE;
import static org.apache.dolphinscheduler.common.constants.Constants.TASK_LIST; import static org.apache.dolphinscheduler.common.constants.Constants.TASK_LIST;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.dto.DynamicSubWorkflowDto; import org.apache.dolphinscheduler.api.dto.DynamicSubWorkflowDto;
@ -77,11 +76,10 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.utils.WorkflowUtils; import org.apache.dolphinscheduler.dao.utils.WorkflowUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.expand.CuringParamsService;
import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
@ -498,7 +496,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId); throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId);
} }
if (!taskInstance.isDynamic()) { if (!TaskTypeUtils.isDynamicTask(taskInstance.getTaskType())) {
putMsg(result, Status.TASK_INSTANCE_NOT_DYNAMIC_TASK, taskInstance.getName()); putMsg(result, Status.TASK_INSTANCE_NOT_DYNAMIC_TASK, taskInstance.getName());
throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId); throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId);
} }
@ -548,7 +546,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
*/ */
private void addDependResultForTaskList(User loginUser, List<TaskInstance> taskInstanceList) throws IOException { private void addDependResultForTaskList(User loginUser, List<TaskInstance> taskInstanceList) throws IOException {
for (TaskInstance taskInstance : taskInstanceList) { for (TaskInstance taskInstance : taskInstanceList) {
if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) { if (TaskTypeUtils.isDependentTask(taskInstance.getTaskType())) {
log.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}", log.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}",
taskInstance.getTaskCode(), taskInstance.getId()); taskInstance.getTaskCode(), taskInstance.getId());
// TODO The result of dependent item should not be obtained from the log, waiting for optimization. // TODO The result of dependent item should not be obtained from the log, waiting for optimization.
@ -628,9 +626,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
return result; return result;
} }
if (!taskInstance.isSubProcess()) { if (!TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) {
log.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.",
TASK_TYPE_SUB_PROCESS, projectCode, taskId);
putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName()); putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName());
return result; return result;
} }
@ -714,11 +710,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
return result; return result;
} }
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
.taskType(taskDefinitionLog.getTaskType())
.taskParams(taskDefinitionLog.getTaskParams())
.dependence(taskDefinitionLog.getDependence())
.build())) {
log.error("Task parameters are invalid, taskDefinitionName:{}.", taskDefinitionLog.getName()); log.error("Task parameters are invalid, taskDefinitionName:{}.", taskDefinitionLog.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
return result; return result;

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java

@ -18,9 +18,6 @@
package org.apache.dolphinscheduler.api.service.impl; package org.apache.dolphinscheduler.api.service.impl;
import static java.util.stream.Collectors.toSet; import static java.util.stream.Collectors.toSet;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationCreateRequest; import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationCreateRequest;
import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationFilterRequest; import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationFilterRequest;
@ -46,6 +43,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
@ -354,9 +352,9 @@ public class ProcessTaskRelationServiceImpl extends BaseServiceImpl implements P
} }
updateProcessDefiniteVersion(loginUser, result, processDefinition); updateProcessDefiniteVersion(loginUser, result, processDefinition);
updateRelation(loginUser, result, processDefinition, processTaskRelationList); updateRelation(loginUser, result, processDefinition, processTaskRelationList);
if (TASK_TYPE_CONDITIONS.equals(taskDefinition.getTaskType()) if (TaskTypeUtils.isConditionTask(taskDefinition.getTaskType())
|| TASK_TYPE_DEPENDENT.equals(taskDefinition.getTaskType()) || TaskTypeUtils.isSubWorkflowTask(taskDefinition.getTaskType())
|| TASK_TYPE_SUB_PROCESS.equals(taskDefinition.getTaskType())) { || TaskTypeUtils.isDependentTask(taskDefinition.getTaskType())) {
int deleteTaskDefinition = taskDefinitionMapper.deleteByCode(taskCode); int deleteTaskDefinition = taskDefinitionMapper.deleteByCode(taskCode);
if (0 == deleteTaskDefinition) { if (0 == deleteTaskDefinition) {
log.error("Delete task definition error, taskDefinitionCode:{}.", taskCode); log.error("Delete task definition error, taskDefinitionCode:{}.", taskCode);

30
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_VERSION_VIEW; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_VERSION_VIEW;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_DEFINITION; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_DEFINITION;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION;
import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest; import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest;
import org.apache.dolphinscheduler.api.dto.task.TaskFilterRequest; import org.apache.dolphinscheduler.api.dto.task.TaskFilterRequest;
@ -67,8 +68,6 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao; import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
@ -167,11 +166,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
return result; return result;
} }
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
.taskType(taskDefinitionLog.getTaskType())
.taskParams(taskDefinitionLog.getTaskParams())
.dependence(taskDefinitionLog.getDependence())
.build())) {
log.warn("Task definition {} parameters are invalid.", taskDefinitionLog.getName()); log.warn("Task definition {} parameters are invalid.", taskDefinitionLog.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
return result; return result;
@ -208,11 +203,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
Project project = projectMapper.queryByCode(taskDefinition.getProjectCode()); Project project = projectMapper.queryByCode(taskDefinition.getProjectCode());
projectService.checkProjectAndAuthThrowException(user, project, permissions); projectService.checkProjectAndAuthThrowException(user, project, permissions);
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) {
.taskType(taskDefinition.getTaskType())
.taskParams(taskDefinition.getTaskParams())
.dependence(taskDefinition.getDependence())
.build())) {
throw new ServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName()); throw new ServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
} }
} }
@ -321,12 +312,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
return result; return result;
} }
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) {
.taskType(taskDefinition.getTaskType())
.taskParams(taskDefinition.getTaskParams())
.dependence(taskDefinition.getDependence())
.build())) {
log.error("Task definition {} parameters are invalid", taskDefinition.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
return result; return result;
} }
@ -732,13 +718,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
return null; return null;
} }
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() if (!checkTaskParameters(taskDefinitionToUpdate.getTaskType(), taskDefinitionToUpdate.getTaskParams())) {
.taskType(taskDefinitionToUpdate.getTaskType())
.taskParams(taskDefinitionToUpdate.getTaskParams())
.dependence(taskDefinitionToUpdate.getDependence())
.build())) {
log.warn("Task definition parameters are invalid, taskDefinitionName:{}.",
taskDefinitionToUpdate.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName());
return null; return null;
} }

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.api.service.impl; package org.apache.dolphinscheduler.api.service.impl;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
@ -39,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -147,7 +146,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
List<Long> processDefinitionCodes) { List<Long> processDefinitionCodes) {
for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) { for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) {
for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters() for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters()
.getDependTaskList()) { .getDependence().getDependTaskList()) {
for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) {
if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) { if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) {
processDefinitionCodes.add(dependentItem.getDefinitionCode()); processDefinitionCodes.add(dependentItem.getDefinitionCode());
@ -220,12 +219,12 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
List<TaskDefinitionLog> taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList); List<TaskDefinitionLog> taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList);
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (taskDefinitionLog.getProjectCode() == projectCode) { if (taskDefinitionLog.getProjectCode() == projectCode) {
if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) { if (TaskTypeUtils.isDependentTask(taskDefinitionLog.getTaskType())) {
DependentParameters dependentParameters = DependentParameters dependentParameters =
JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class);
if (dependentParameters != null) { if (dependentParameters != null) {
List<DependentTaskModel> dependTaskList = List<DependentTaskModel> dependTaskList =
dependentParameters.getDependTaskList(); dependentParameters.getDependence().getDependTaskList();
if (!CollectionUtils.isEmpty(dependTaskList)) { if (!CollectionUtils.isEmpty(dependTaskList)) {
for (DependentTaskModel taskModel : dependTaskList) { for (DependentTaskModel taskModel : dependTaskList) {
List<DependentItem> dependItemList = taskModel.getDependItemList(); List<DependentItem> dependItemList = taskModel.getDependItemList();

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java

@ -627,7 +627,8 @@ public class ProcessInstanceServiceTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true); .thenReturn(true);
Map<String, Object> processInstanceFinishRes = Map<String, Object> processInstanceFinishRes =
processInstanceService.updateProcessInstance(loginUser, projectCode, 1, processInstanceService.updateProcessInstance(loginUser, projectCode, 1,

22
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java

@ -64,6 +64,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao; import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.process.ProcessServiceImpl; import org.apache.dolphinscheduler.service.process.ProcessServiceImpl;
@ -165,7 +166,8 @@ public class TaskDefinitionServiceImplTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true); .thenReturn(true);
Project project = getProject(); Project project = getProject();
when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project);
@ -194,7 +196,8 @@ public class TaskDefinitionServiceImplTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true); .thenReturn(true);
String taskDefinitionJson = getTaskDefinitionJson(); String taskDefinitionJson = getTaskDefinitionJson();
@ -441,7 +444,8 @@ public class TaskDefinitionServiceImplTest {
() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest));
// error task definition // error task definition
taskCreateRequest.setTaskParams(TASK_PARAMETER); taskCreateRequest.setTaskType(ShellTaskChannelFactory.NAME);
taskCreateRequest.setTaskParams(JSONUtils.toJsonString(new HashMap<>()));
doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE);
assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID,
() -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest));
@ -449,7 +453,8 @@ public class TaskDefinitionServiceImplTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true); .thenReturn(true);
// error create task definition object // error create task definition object
@ -502,7 +507,8 @@ public class TaskDefinitionServiceImplTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(false); .thenReturn(false);
assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID,
() -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest));
@ -511,7 +517,8 @@ public class TaskDefinitionServiceImplTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true); .thenReturn(true);
// error task definition nothing update // error task definition nothing update
when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); when(processService.isTaskOnline(TASK_CODE)).thenReturn(false);
@ -616,7 +623,8 @@ public class TaskDefinitionServiceImplTest {
try ( try (
MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic = MockedStatic<TaskPluginManager> taskPluginManagerMockedStatic =
Mockito.mockStatic(TaskPluginManager.class)) { Mockito.mockStatic(TaskPluginManager.class)) {
taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) taskPluginManagerMockedStatic
.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any()))
.thenReturn(true); .thenReturn(true);
String taskDefinitionJson = getTaskDefinitionJson(); String taskDefinitionJson = getTaskDefinitionJson();
TaskDefinition taskDefinition = getTaskDefinition(); TaskDefinition taskDefinition = getTaskDefinition();

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java

@ -272,8 +272,6 @@ public final class Constants {
*/ */
public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL"; public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL";
public static final String COMMON_TASK_TYPE = "common";
public static final String DEFAULT = "default"; public static final String DEFAULT = "default";
public static final String PASSWORD = "password"; public static final String PASSWORD = "password";
public static final String XXXXXX = "******"; public static final String XXXXXX = "******";

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java

@ -27,7 +27,6 @@ public enum StateEventType {
TASK_TIMEOUT(3, "task timeout"), TASK_TIMEOUT(3, "task timeout"),
WAKE_UP_TASK_GROUP(4, "wait task group"), WAKE_UP_TASK_GROUP(4, "wait task group"),
TASK_RETRY(5, "task retry"), TASK_RETRY(5, "task retry"),
PROCESS_BLOCKED(6, "process blocked"),
PROCESS_SUBMIT_FAILED(7, "process submit failed"); PROCESS_SUBMIT_FAILED(7, "process submit failed");
StateEventType(int code, String descp) { StateEventType(int code, String descp) {

9
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java

@ -38,8 +38,6 @@ public enum WorkflowExecutionStatus {
SUCCESS(7, "success"), SUCCESS(7, "success"),
DELAY_EXECUTION(12, "delay execution"), DELAY_EXECUTION(12, "delay execution"),
SERIAL_WAIT(14, "serial wait"), SERIAL_WAIT(14, "serial wait"),
READY_BLOCK(15, "ready block"),
BLOCK(16, "block"),
WAIT_TO_RUN(17, "wait to run"), WAIT_TO_RUN(17, "wait to run"),
; ;
@ -59,7 +57,6 @@ public enum WorkflowExecutionStatus {
READY_PAUSE.getCode(), READY_PAUSE.getCode(),
READY_STOP.getCode(), READY_STOP.getCode(),
SERIAL_WAIT.getCode(), SERIAL_WAIT.getCode(),
READY_BLOCK.getCode(),
WAIT_TO_RUN.getCode() WAIT_TO_RUN.getCode()
}; };
@ -91,7 +88,7 @@ public enum WorkflowExecutionStatus {
public boolean isFinished() { public boolean isFinished() {
// todo: do we need to remove pause/block in finished judge? // todo: do we need to remove pause/block in finished judge?
return isSuccess() || isFailure() || isStop() || isPause() || isBlock(); return isSuccess() || isFailure() || isStop() || isPause();
} }
/** /**
@ -119,10 +116,6 @@ public enum WorkflowExecutionStatus {
return this == STOP; return this == STOP;
} }
public boolean isBlock() {
return this == BLOCK;
}
public static int[] getNeedFailoverWorkflowInstanceState() { public static int[] getNeedFailoverWorkflowInstanceState() {
return NEED_FAILOVER_STATES; return NEED_FAILOVER_STATES;
} }

9
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.enums.CycleEnum;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
@ -67,7 +66,7 @@ public class DependentProcessDefinition {
*/ */
public CycleEnum getDependentCycle(long upstreamProcessDefinitionCode) { public CycleEnum getDependentCycle(long upstreamProcessDefinitionCode) {
DependentParameters dependentParameters = this.getDependentParameters(); DependentParameters dependentParameters = this.getDependentParameters();
List<DependentTaskModel> dependentTaskModelList = dependentParameters.getDependTaskList(); List<DependentTaskModel> dependentTaskModelList = dependentParameters.getDependence().getDependTaskList();
for (DependentTaskModel dependentTaskModel : dependentTaskModelList) { for (DependentTaskModel dependentTaskModel : dependentTaskModelList) {
List<DependentItem> dependentItemList = dependentTaskModel.getDependItemList(); List<DependentItem> dependentItemList = dependentTaskModel.getDependItemList();
@ -104,11 +103,7 @@ public class DependentProcessDefinition {
} }
public DependentParameters getDependentParameters() { public DependentParameters getDependentParameters() {
return JSONUtils.parseObject(getDependence(), DependentParameters.class); return JSONUtils.parseObject(taskParams, DependentParameters.class);
}
public String getDependence() {
return JSONUtils.getNodeString(this.taskParams, Constants.DEPENDENCE);
} }
public String getProcessDefinitionName() { public String getProcessDefinitionName() {

6
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java

@ -201,12 +201,6 @@ public class ProcessInstance {
*/ */
private Date restartTime; private Date restartTime;
/**
* workflow block flag
*/
@TableField(exist = false)
private boolean isBlocked;
/** /**
* test flag * test flag
*/ */

133
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java

@ -17,30 +17,15 @@
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DYNAMIC;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.enums.TaskExecuteType;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable; import java.io.Serializable;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import lombok.Data; import lombok.Data;
@ -50,7 +35,6 @@ import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.core.type.TypeReference;
/** /**
* task instance * task instance
@ -190,21 +174,6 @@ public class TaskInstance implements Serializable {
@TableField(updateStrategy = FieldStrategy.IGNORED) @TableField(updateStrategy = FieldStrategy.IGNORED)
private String cacheKey; private String cacheKey;
/**
* dependency
*/
@TableField(exist = false)
private DependentParameters dependency;
@TableField(exist = false)
private ConditionsParameters conditionsParameters;
/**
* switch dependency
*/
@TableField(exist = false)
private SwitchParameters switchDependency;
/** /**
* duration * duration
*/ */
@ -310,80 +279,6 @@ public class TaskInstance implements Serializable {
this.executePath = executePath; this.executePath = executePath;
} }
public DependentParameters getDependency() {
if (this.dependency == null) {
Map<String, Object> taskParamsMap =
JSONUtils.parseObject(this.getTaskParams(), new TypeReference<Map<String, Object>>() {
});
this.dependency =
JSONUtils.parseObject((String) taskParamsMap.get(Constants.DEPENDENCE), DependentParameters.class);
}
return this.dependency;
}
public void setDependency(DependentParameters dependency) {
this.dependency = dependency;
}
public ConditionsParameters getConditionsParameters() {
if (this.conditionsParameters == null) {
Map<String, Object> taskParamsMap =
JSONUtils.parseObject(this.getTaskParams(), new TypeReference<Map<String, Object>>() {
});
this.conditionsParameters =
JSONUtils.parseObject((String) taskParamsMap.get(Constants.DEPENDENCE), ConditionsParameters.class);
}
return conditionsParameters;
}
public ConditionsParameters.ConditionResult getConditionResult() {
Map<String, Object> taskParamsMap =
JSONUtils.parseObject(this.getTaskParams(), new TypeReference<Map<String, Object>>() {
});
String conditionResult = (String) taskParamsMap.getOrDefault(Constants.CONDITION_RESULT, "");
if (StringUtils.isNotEmpty(conditionResult)) {
return JSONUtils.parseObject(conditionResult, new TypeReference<ConditionsParameters.ConditionResult>() {
});
}
return null;
}
public void setConditionResult(ConditionsParameters conditionsParameters) {
if (conditionsParameters == null) {
return;
}
Map<String, Object> taskParamsMap =
JSONUtils.parseObject(this.getTaskParams(), new TypeReference<Map<String, Object>>() {
});
if (taskParamsMap == null) {
taskParamsMap = new HashMap<>();
}
taskParamsMap.put(Constants.CONDITION_RESULT, JSONUtils.toJsonString(conditionsParameters));
this.setTaskParams(JSONUtils.toJsonString(taskParamsMap));
}
public SwitchParameters getSwitchDependency() {
// todo: We need to directly use Jackson to deserialize the taskParam, rather than parse the map and get from
// field.
if (this.switchDependency == null) {
Map<String, Object> taskParamsMap =
JSONUtils.parseObject(this.getTaskParams(), new TypeReference<Map<String, Object>>() {
});
this.switchDependency =
JSONUtils.parseObject((String) taskParamsMap.get(Constants.SWITCH_RESULT), SwitchParameters.class);
}
return this.switchDependency;
}
public void setSwitchDependency(SwitchParameters switchDependency) {
Map<String, Object> taskParamsMap =
JSONUtils.parseObject(this.getTaskParams(), new TypeReference<Map<String, Object>>() {
});
taskParamsMap.put(Constants.SWITCH_RESULT, JSONUtils.toJsonString(switchDependency));
this.switchDependency = switchDependency;
this.setTaskParams(JSONUtils.toJsonString(taskParamsMap));
}
public boolean isTaskComplete() { public boolean isTaskComplete() {
return this.getState().isSuccess() return this.getState().isSuccess()
@ -392,30 +287,6 @@ public class TaskInstance implements Serializable {
|| this.getState().isForceSuccess(); || this.getState().isForceSuccess();
} }
public boolean isSubProcess() {
return TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(this.taskType);
}
public boolean isDependTask() {
return TASK_TYPE_DEPENDENT.equalsIgnoreCase(this.taskType);
}
public boolean isDynamic() {
return TASK_TYPE_DYNAMIC.equalsIgnoreCase(this.taskType);
}
public boolean isConditionsTask() {
return TASK_TYPE_CONDITIONS.equalsIgnoreCase(this.taskType);
}
public boolean isSwitchTask() {
return TASK_TYPE_SWITCH.equalsIgnoreCase(this.taskType);
}
public boolean isBlockingTask() {
return TASK_TYPE_BLOCKING.equalsIgnoreCase(this.taskType);
}
public boolean isFirstRun() { public boolean isFirstRun() {
return endTime == null; return endTime == null;
} }
@ -427,7 +298,7 @@ public class TaskInstance implements Serializable {
* @return can try result * @return can try result
*/ */
public boolean taskCanRetry() { public boolean taskCanRetry() {
if (this.isSubProcess()) { if (TaskTypeUtils.isSubWorkflowTask(getTaskType())) {
return false; return false;
} }
if (this.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE) { if (this.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE) {

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java

@ -55,11 +55,9 @@ public class TaskInstanceUtils {
target.setPid(source.getPid()); target.setPid(source.getPid());
target.setAppLink(source.getAppLink()); target.setAppLink(source.getAppLink());
target.setFlag(source.getFlag()); target.setFlag(source.getFlag());
target.setDependency(source.getDependency());
// todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on
// task params, this is really a very bad practice. // task params, this is really a very bad practice.
target.setTaskParams(source.getTaskParams()); target.setTaskParams(source.getTaskParams());
target.setSwitchDependency(source.getSwitchDependency());
target.setDuration(source.getDuration()); target.setDuration(source.getDuration());
target.setMaxRetryTimes(source.getMaxRetryTimes()); target.setMaxRetryTimes(source.getMaxRetryTimes());
target.setRetryInterval(source.getRetryInterval()); target.setRetryInterval(source.getRetryInterval());

33
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java

@ -17,12 +17,7 @@
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.dao.entity;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
@ -30,35 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameter
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
public class TaskInstanceTest { public class TaskInstanceTest {
/**
* task instance sub process
*/
@Test
public void testTaskInstanceIsSubProcess() {
TaskInstance taskInstance = new TaskInstance();
// sub process
taskInstance.setTaskType(TASK_TYPE_SUB_PROCESS);
Assertions.assertTrue(taskInstance.isSubProcess());
// not sub process
taskInstance.setTaskType("HTTP");
Assertions.assertFalse(taskInstance.isSubProcess());
// sub process
taskInstance.setTaskType(TASK_TYPE_CONDITIONS);
Assertions.assertTrue(taskInstance.isConditionsTask());
// sub process
taskInstance.setTaskType(TASK_TYPE_DEPENDENT);
Assertions.assertTrue(taskInstance.isDependTask());
}
/** /**
* test for TaskInstance.getDependence * test for TaskInstance.getDependence
*/ */
@ -66,7 +36,6 @@ public class TaskInstanceTest {
public void testTaskInstanceGetDependence() { public void testTaskInstanceGetDependence() {
TaskInstance taskInstance = new TaskInstance(); TaskInstance taskInstance = new TaskInstance();
taskInstance.setTaskParams(JSONUtils.toJsonString(getDependentParameters())); taskInstance.setTaskParams(JSONUtils.toJsonString(getDependentParameters()));
taskInstance.getDependency();
} }
/** /**
@ -82,8 +51,6 @@ public class TaskInstanceTest {
dependentItem.setDefinitionCode(222L); dependentItem.setDefinitionCode(222L);
dependentItem.setCycle("today"); dependentItem.setCycle("today");
dependentItems.add(dependentItem); dependentItems.add(dependentItem);
dependentParameters.setDependTaskList(dependTaskList);
dependentParameters.setRelation(DependentRelation.AND);
return dependentParameters; return dependentParameters;
} }
} }

4
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java

@ -65,11 +65,9 @@ class ProcessInstanceDaoImplTest extends BaseDaoTest {
WorkflowExecutionStatus.READY_STOP)); WorkflowExecutionStatus.READY_STOP));
processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion, processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion,
WorkflowExecutionStatus.SERIAL_WAIT)); WorkflowExecutionStatus.SERIAL_WAIT));
processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion,
WorkflowExecutionStatus.READY_BLOCK));
processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion, processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion,
WorkflowExecutionStatus.WAIT_TO_RUN)); WorkflowExecutionStatus.WAIT_TO_RUN));
assertEquals(8, processInstanceDao assertEquals(7, processInstanceDao
.queryByWorkflowCodeVersionStatus(workflowDefinitionCode, workflowDefinitionVersion, status).size()); .queryByWorkflowCodeVersionStatus(workflowDefinitionCode, workflowDefinitionVersion, status).size());
} }

21
dolphinscheduler-dao/src/test/resources/logback.xml

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration scan="true" scanPeriod="120 seconds">
<logger name="*" level="ERROR"/>
</configuration>

2
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java

@ -108,7 +108,7 @@ public class MasterServer implements IStoppable {
this.masterRPCServer.start(); this.masterRPCServer.start();
// install task plugin // install task plugin
TaskPluginManager.loadPlugin(); TaskPluginManager.loadTaskPlugin();
DataSourceProcessorProvider.initialize(); DataSourceProcessorProvider.initialize();
this.masterSlotManager.start(); this.masterSlotManager.start();

59
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java

@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.event;
import org.apache.dolphinscheduler.common.enums.StateEventType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import com.google.auto.service.AutoService;
@AutoService(StateEventHandler.class)
@Slf4j
public class WorkflowBlockStateEventHandler implements StateEventHandler {
@Override
public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
StateEvent stateEvent) throws StateEventHandleError {
log.info("Handle workflow instance state block event");
Optional<TaskInstance> taskInstanceOptional =
workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId());
if (!taskInstanceOptional.isPresent()) {
throw new StateEventHandleError("Cannot find taskInstance from taskMap by taskInstanceId: "
+ stateEvent.getTaskInstanceId());
}
TaskInstance task = taskInstanceOptional.get();
BlockingParameters parameters = JSONUtils.parseObject(task.getTaskParams(), BlockingParameters.class);
if (parameters != null && parameters.isAlertWhenBlocking()) {
workflowExecuteRunnable.processBlock();
}
return true;
}
@Override
public StateEventType getEventType() {
return StateEventType.PROCESS_BLOCKED;
}
}

15
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java

@ -43,7 +43,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
@ -89,8 +88,6 @@ public class StreamTaskExecuteRunnable implements Runnable {
protected ProcessTaskRelationMapper processTaskRelationMapper; protected ProcessTaskRelationMapper processTaskRelationMapper;
protected TaskPluginManager taskPluginManager;
private StreamTaskInstanceExecCacheManager streamTaskInstanceExecCacheManager; private StreamTaskInstanceExecCacheManager streamTaskInstanceExecCacheManager;
protected TaskDefinition taskDefinition; protected TaskDefinition taskDefinition;
@ -115,7 +112,6 @@ public class StreamTaskExecuteRunnable implements Runnable {
this.processService = SpringApplicationContext.getBean(ProcessService.class); this.processService = SpringApplicationContext.getBean(ProcessService.class);
this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class); this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
this.workerTaskDispatcher = SpringApplicationContext.getBean(WorkerTaskDispatcher.class); this.workerTaskDispatcher = SpringApplicationContext.getBean(WorkerTaskDispatcher.class);
this.taskPluginManager = SpringApplicationContext.getBean(TaskPluginManager.class);
this.processTaskRelationMapper = SpringApplicationContext.getBean(ProcessTaskRelationMapper.class); this.processTaskRelationMapper = SpringApplicationContext.getBean(ProcessTaskRelationMapper.class);
this.taskInstanceDao = SpringApplicationContext.getBean(TaskInstanceDao.class); this.taskInstanceDao = SpringApplicationContext.getBean(TaskInstanceDao.class);
this.streamTaskInstanceExecCacheManager = this.streamTaskInstanceExecCacheManager =
@ -311,14 +307,11 @@ public class StreamTaskExecuteRunnable implements Runnable {
return null; return null;
} }
TaskChannel taskChannel = taskPluginManager.getTaskChannel(taskInstance.getTaskType()); TaskChannel taskChannel = TaskPluginManager.getTaskChannel(taskInstance.getTaskType());
ResourceParametersHelper resources = taskChannel.getResources(taskInstance.getTaskParams()); ResourceParametersHelper resources = taskChannel.parseParameters(taskInstance.getTaskParams()).getResources();
AbstractParameters baseParam = taskPluginManager.getParameters( AbstractParameters baseParam =
ParametersNode.builder() TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams());
.taskType(taskInstance.getTaskType())
.taskParams(taskInstance.getTaskParams())
.build());
Map<String, Property> propertyMap = paramParsingPreparation(taskInstance, baseParam); Map<String, Property> propertyMap = paramParsingPreparation(taskInstance, baseParam);
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
.buildWorkflowInstanceHost(masterConfig.getMasterAddress()) .buildWorkflowInstanceHost(masterConfig.getMasterAddress())

13
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java

@ -49,7 +49,6 @@ import org.apache.dolphinscheduler.plugin.task.api.model.JdbcInfo;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.AbstractResourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.AbstractResourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
@ -74,7 +73,6 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -102,16 +100,15 @@ public class TaskExecutionContextFactory {
public TaskExecutionContext createTaskExecutionContext(TaskInstance taskInstance) throws TaskExecutionContextCreateException { public TaskExecutionContext createTaskExecutionContext(TaskInstance taskInstance) throws TaskExecutionContextCreateException {
ProcessInstance workflowInstance = taskInstance.getProcessInstance(); ProcessInstance workflowInstance = taskInstance.getProcessInstance();
ResourceParametersHelper resources = ResourceParametersHelper resources = TaskPluginManager.getTaskChannel(taskInstance.getTaskType())
Optional.ofNullable(TaskPluginManager.getTaskChannel(taskInstance.getTaskType())) .parseParameters(taskInstance.getTaskParams())
.map(taskChannel -> taskChannel.getResources(taskInstance.getTaskParams())) .getResources();
.orElse(null);
setTaskResourceInfo(resources); setTaskResourceInfo(resources);
Map<String, Property> businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance); Map<String, Property> businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance);
AbstractParameters baseParam = TaskPluginManager.getParameters(ParametersNode.builder() AbstractParameters baseParam =
.taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build()); TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams());
Map<String, Property> propertyMap = Map<String, Property> propertyMap =
curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance); curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance);
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()

77
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java

@ -28,7 +28,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.COMMA;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM_DD_HH_MM_SS; import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM_DD_HH_MM_SS;
import static org.apache.dolphinscheduler.dao.utils.EnvironmentUtils.getEnvironmentCodeOrDefault; import static org.apache.dolphinscheduler.dao.utils.EnvironmentUtils.getEnvironmentCodeOrDefault;
import static org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils.getWorkerGroupOrDefault; import static org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils.getWorkerGroupOrDefault;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
@ -61,8 +60,8 @@ import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHost
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.StateEvent;
@ -76,7 +75,6 @@ import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph;
import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator; import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator;
import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils; import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager; import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
@ -300,7 +298,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
stateEvent, stateEvent,
stateEventHandleException); stateEventHandleException);
ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (Exception e) { } catch (Throwable e) {
// we catch the exception here, since if the state event handle failed, the state event will still // we catch the exception here, since if the state event handle failed, the state event will still
// keep // keep
// in the stateEvents queue. // in the stateEvents queue.
@ -381,9 +379,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
if (taskInstance.getIsCache().equals(Flag.YES)) { if (taskInstance.getIsCache().equals(Flag.YES)) {
saveCacheTaskInstance(taskInstance); saveCacheTaskInstance(taskInstance);
} }
if (!workflowInstance.isBlocked()) {
submitPostNode(taskInstance.getTaskCode()); submitPostNode(taskInstance.getTaskCode());
}
} else if (taskInstance.taskCanRetry() && !workflowInstance.getState().isReadyStop()) { } else if (taskInstance.taskCanRetry() && !workflowInstance.getState().isReadyStop()) {
// retry task // retry task
log.info("Retry taskInstance taskInstance state: {}", taskInstance.getState()); log.info("Retry taskInstance taskInstance state: {}", taskInstance.getState());
@ -950,16 +946,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
if (!taskInstance.getState().isFinished()) { if (!taskInstance.getState().isFinished()) {
taskExecuteRunnable.dispatch(); taskExecuteRunnable.dispatch();
} else { } else {
if (workflowExecuteContext.getWorkflowInstance().isBlocked()) {
TaskStateEvent processBlockEvent = TaskStateEvent.builder()
.processInstanceId(workflowExecuteContext.getWorkflowInstance().getId())
.taskInstanceId(taskInstance.getId())
.status(taskInstance.getState())
.type(StateEventType.PROCESS_BLOCKED)
.build();
this.stateEvents.add(processBlockEvent);
}
TaskStateEvent taskStateChangeEvent = TaskStateEvent.builder() TaskStateEvent taskStateChangeEvent = TaskStateEvent.builder()
.processInstanceId(workflowExecuteContext.getWorkflowInstance().getId()) .processInstanceId(workflowExecuteContext.getWorkflowInstance().getId())
.taskInstanceId(taskInstance.getId()) .taskInstanceId(taskInstance.getId())
@ -1096,7 +1082,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
taskInstance.setRetryInterval(taskNode.getRetryInterval()); taskInstance.setRetryInterval(taskNode.getRetryInterval());
// set task param // set task param
taskInstance.setTaskParams(taskNode.getTaskParams()); taskInstance.setTaskParams(taskNode.getParams());
// set task group and priority // set task group and priority
taskInstance.setTaskGroupId(taskNode.getTaskGroupId()); taskInstance.setTaskGroupId(taskNode.getTaskGroupId());
@ -1275,7 +1261,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
private boolean tryToTakeOverTaskInstance(TaskInstance taskInstance) { private boolean tryToTakeOverTaskInstance(TaskInstance taskInstance) {
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance(); ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
if (TaskUtils.isMasterTask(taskInstance.getTaskType())) { if (TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) {
return false; return false;
} }
try { try {
@ -1341,13 +1327,14 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
if (depTaskState.isKill()) { if (depTaskState.isKill()) {
return DependResult.NON_EXEC; return DependResult.NON_EXEC;
} }
// ignore task state if current task is block
if (taskNode.isBlockingTask()) { // always return success if current task is condition
if (TaskTypeUtils.isConditionTask(taskNode.getType())) {
continue; continue;
} }
// always return success if current task is condition // always return success if current task is switch
if (taskNode.isConditionsTask()) { if (TaskTypeUtils.isSwitchTask(taskNode.getType())) {
continue; continue;
} }
@ -1392,7 +1379,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
private boolean dependTaskSuccess(Long dependNodeCode, Long nextNodeCode) { private boolean dependTaskSuccess(Long dependNodeCode, Long nextNodeCode) {
DAG<Long, TaskNode, TaskNodeRelation> dag = workflowExecuteContext.getWorkflowGraph().getDag(); DAG<Long, TaskNode, TaskNodeRelation> dag = workflowExecuteContext.getWorkflowGraph().getDag();
TaskNode dependentNode = dag.getNode(dependNodeCode); TaskNode dependentNode = dag.getNode(dependNodeCode);
if (dependentNode.isConditionsTask()) { if (TaskTypeUtils.isConditionTask(dependentNode.getType())) {
// condition task need check the branch to run // condition task need check the branch to run
List<Long> nextTaskList = List<Long> nextTaskList =
DagHelper.parseConditionTask(dependNodeCode, skipTaskNodeMap, dag, getCompleteTaskInstanceMap()); DagHelper.parseConditionTask(dependNodeCode, skipTaskNodeMap, dag, getCompleteTaskInstanceMap());
@ -1405,12 +1392,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
} }
return true; return true;
} }
if (dependentNode.isSwitchTask()) {
TaskInstance dependentTaskInstance = taskInstanceMap.get(validTaskMap.get(dependentNode.getCode()));
SwitchParameters switchParameters = dependentTaskInstance.getSwitchDependency();
return switchParameters.getDependTaskList().get(switchParameters.getResultConditionLocation()).getNextNode()
.contains(nextNodeCode);
}
Optional<TaskInstance> existTaskInstanceOptional = getTaskInstance(dependNodeCode); Optional<TaskInstance> existTaskInstanceOptional = getTaskInstance(dependNodeCode);
if (!existTaskInstanceOptional.isPresent()) { if (!existTaskInstanceOptional.isPresent()) {
return false; return false;
@ -1450,8 +1431,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
*/ */
private WorkflowExecutionStatus runningState(WorkflowExecutionStatus state) { private WorkflowExecutionStatus runningState(WorkflowExecutionStatus state) {
if (state == WorkflowExecutionStatus.READY_STOP || state == WorkflowExecutionStatus.READY_PAUSE if (state == WorkflowExecutionStatus.READY_STOP || state == WorkflowExecutionStatus.READY_PAUSE
|| state == WorkflowExecutionStatus.READY_BLOCK || || state == WorkflowExecutionStatus.DELAY_EXECUTION) {
state == WorkflowExecutionStatus.DELAY_EXECUTION) {
// if the running task is not completed, the state remains unchanged // if the running task is not completed, the state remains unchanged
return state; return state;
} else { } else {
@ -1510,7 +1490,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
} }
List<TaskInstance> pauseList = getCompleteTaskByState(TaskExecutionStatus.PAUSE); List<TaskInstance> pauseList = getCompleteTaskByState(TaskExecutionStatus.PAUSE);
if (CollectionUtils.isNotEmpty(pauseList) || workflowInstance.isBlocked() || !isComplementEnd() if (CollectionUtils.isNotEmpty(pauseList) || !isComplementEnd()
|| standByTaskInstancePriorityQueue.size() > 0) { || standByTaskInstancePriorityQueue.size() > 0) {
return WorkflowExecutionStatus.PAUSE; return WorkflowExecutionStatus.PAUSE;
} else { } else {
@ -1518,30 +1498,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
} }
} }
/**
* prepare for block
* if process has tasks still running, pause them
* if readyToSubmitTaskQueue is not empty, kill them
* else return block status directly
*
* @return ExecutionStatus
*/
private WorkflowExecutionStatus processReadyBlock() {
if (taskExecuteRunnableMap.size() > 0) {
for (DefaultTaskExecuteRunnable taskExecuteRunnable : taskExecuteRunnableMap.values()) {
if (!TASK_TYPE_BLOCKING.equals(taskExecuteRunnable.getTaskInstance().getTaskType())) {
taskExecuteRunnable.pause();
}
}
}
if (standByTaskInstancePriorityQueue.size() > 0) {
for (Iterator<TaskInstance> iter = standByTaskInstancePriorityQueue.iterator(); iter.hasNext();) {
iter.next().setState(TaskExecutionStatus.PAUSE);
}
}
return WorkflowExecutionStatus.BLOCK;
}
/** /**
* generate the latest process instance status by the tasks state * generate the latest process instance status by the tasks state
* *
@ -1557,13 +1513,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
return executionStatus; return executionStatus;
} }
// block
if (state == WorkflowExecutionStatus.READY_BLOCK) {
WorkflowExecutionStatus executionStatus = processReadyBlock();
log.info("The workflowInstance is ready to block, the workflowInstance status is {}", executionStatus);
return executionStatus;
}
// pause // pause
if (state == WorkflowExecutionStatus.READY_PAUSE) { if (state == WorkflowExecutionStatus.READY_PAUSE) {
WorkflowExecutionStatus executionStatus = processReadyPause(); WorkflowExecutionStatus executionStatus = processReadyPause();
@ -2038,7 +1987,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
} }
private void sendTaskLogOnMasterToRemoteIfNeeded(TaskInstance taskInstance) { private void sendTaskLogOnMasterToRemoteIfNeeded(TaskInstance taskInstance) {
if (RemoteLogUtils.isRemoteLoggingEnable() && TaskUtils.isMasterTask(taskInstance.getTaskType())) { if (RemoteLogUtils.isRemoteLoggingEnable() && TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) {
RemoteLogUtils.sendRemoteLog(taskInstance.getLogPath()); RemoteLogUtils.sendRemoteLog(taskInstance.getLogPath());
log.info("Master sends task log {} to remote storage asynchronously.", taskInstance.getLogPath()); log.info("Master sends task log {} to remote storage asynchronously.", taskInstance.getLogPath());
} }

7
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java

@ -18,7 +18,7 @@
package org.apache.dolphinscheduler.server.master.runner.dispatcher; package org.apache.dolphinscheduler.server.master.runner.dispatcher;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.master.utils.TaskUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -36,7 +36,10 @@ public class TaskDispatchFactory {
private WorkerTaskDispatcher workerTaskDispatcher; private WorkerTaskDispatcher workerTaskDispatcher;
public TaskDispatcher getTaskDispatcher(String taskType) { public TaskDispatcher getTaskDispatcher(String taskType) {
return TaskUtils.isMasterTask(taskType) ? masterTaskDispatcher : workerTaskDispatcher; if (TaskTypeUtils.isLogicTask(taskType)) {
return masterTaskDispatcher;
}
return workerTaskDispatcher;
} }
public TaskDispatcher getTaskDispatcher(TaskInstance taskInstance) { public TaskDispatcher getTaskDispatcher(TaskInstance taskInstance) {

11
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java

@ -17,8 +17,8 @@
package org.apache.dolphinscheduler.server.master.runner.operator; package org.apache.dolphinscheduler.server.master.runner.operator;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable;
import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -51,28 +51,29 @@ public class TaskExecuteRunnableOperatorManager {
private LogicTaskExecuteRunnableTimeoutOperator logicTaskTimeoutOperator; private LogicTaskExecuteRunnableTimeoutOperator logicTaskTimeoutOperator;
public TaskExecuteRunnableOperator getTaskKillOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { public TaskExecuteRunnableOperator getTaskKillOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskKillOperator; return logicTaskKillOperator;
} }
return taskKillOperator; return taskKillOperator;
} }
public TaskExecuteRunnableOperator getTaskPauseOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { public TaskExecuteRunnableOperator getTaskPauseOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskPauseOperator; return logicTaskPauseOperator;
} }
return taskPauseOperator; return taskPauseOperator;
} }
public TaskExecuteRunnableOperator getTaskDispatchOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { public TaskExecuteRunnableOperator getTaskDispatchOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskDispatchOperator; return logicTaskDispatchOperator;
} }
return taskDispatchOperator; return taskDispatchOperator;
} }
public TaskExecuteRunnableOperator getTaskTimeoutOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { public TaskExecuteRunnableOperator getTaskTimeoutOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) {
if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) {
return logicTaskTimeoutOperator; return logicTaskTimeoutOperator;
} }
return taskTimeoutOperator; return taskTimeoutOperator;

15
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java

@ -18,9 +18,12 @@
package org.apache.dolphinscheduler.server.master.runner.task; package org.apache.dolphinscheduler.server.master.runner.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -28,10 +31,20 @@ import lombok.extern.slf4j.Slf4j;
public abstract class BaseSyncLogicTask<T extends AbstractParameters> implements ISyncLogicTask { public abstract class BaseSyncLogicTask<T extends AbstractParameters> implements ISyncLogicTask {
protected final TaskExecutionContext taskExecutionContext; protected final TaskExecutionContext taskExecutionContext;
protected final WorkflowExecuteRunnable workflowExecuteRunnable;
protected final TaskInstance taskInstance;
protected final T taskParameters; protected final T taskParameters;
protected BaseSyncLogicTask(TaskExecutionContext taskExecutionContext, T taskParameters) { protected BaseSyncLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable,
TaskExecutionContext taskExecutionContext,
T taskParameters) throws LogicTaskInitializeException {
this.taskExecutionContext = taskExecutionContext; this.taskExecutionContext = taskExecutionContext;
this.workflowExecuteRunnable = workflowExecuteRunnable;
this.taskInstance =
workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId()).orElseThrow(
() -> new LogicTaskInitializeException(
"Cannot find the task instance in workflow execute runnable"));
this.taskParameters = taskParameters; this.taskParameters = taskParameters;
log.info("Success initialize task parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters)); log.info("Success initialize task parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters));
} }

139
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java

@ -1,139 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.runner.task.blocking;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.BlockingOpportunity;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j
public class BlockingLogicTask extends BaseSyncLogicTask<BlockingParameters> {
public static final String TASK_TYPE = "BLOCKING";
private final ProcessInstanceExecCacheManager processInstanceExecCacheManager;
private final ProcessInstanceDao processInstanceDao;
private final TaskInstanceDao taskInstanceDao;
public BlockingLogicTask(TaskExecutionContext taskExecutionContext,
ProcessInstanceExecCacheManager processInstanceExecCacheManager,
ProcessInstanceDao processInstanceDao,
TaskInstanceDao taskInstanceDao) {
super(taskExecutionContext,
JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<BlockingParameters>() {
}));
this.processInstanceExecCacheManager = processInstanceExecCacheManager;
this.processInstanceDao = processInstanceDao;
this.taskInstanceDao = taskInstanceDao;
}
@Override
public void handle() throws MasterTaskExecuteException {
DependResult conditionResult = calculateConditionResult();
DependResult expected = taskParameters.getBlockingOpportunity()
.equals(BlockingOpportunity.BLOCKING_ON_SUCCESS.getDesc())
? DependResult.SUCCESS
: DependResult.FAILED;
boolean isBlocked = (expected == conditionResult);
log.info("blocking opportunity: expected-->{}, actual-->{}", expected, conditionResult);
ProcessInstance workflowInstance = processInstanceExecCacheManager
.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()).getWorkflowExecuteContext()
.getWorkflowInstance();
workflowInstance.setBlocked(isBlocked);
if (isBlocked) {
workflowInstance.setStateWithDesc(WorkflowExecutionStatus.READY_BLOCK, "ready block");
}
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
}
private DependResult calculateConditionResult() throws MasterTaskExecuteException {
// todo: Directly get the task instance from the cache
Map<Long, TaskInstance> completeTaskList = taskInstanceDao
.queryValidTaskListByWorkflowInstanceId(taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTestFlag())
.stream()
.collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity()));
// todo: we need to parse the task parameter from TaskExecutionContext
TaskInstance taskInstance =
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId())
.getTaskInstance(taskExecutionContext.getTaskInstanceId())
.orElseThrow(() -> new MasterTaskExecuteException("Task instance not found"));
DependentParameters dependentParameters = taskInstance.getDependency();
List<DependResult> tempResultList = new ArrayList<>();
for (DependentTaskModel dependentTaskModel : dependentParameters.getDependTaskList()) {
List<DependResult> itemDependResult = new ArrayList<>();
for (DependentItem item : dependentTaskModel.getDependItemList()) {
itemDependResult.add(getDependResultForItem(item, completeTaskList));
}
DependResult tempResult =
DependentUtils.getDependResultForRelation(dependentTaskModel.getRelation(), itemDependResult);
tempResultList.add(tempResult);
}
return DependentUtils.getDependResultForRelation(dependentParameters.getRelation(), tempResultList);
}
private DependResult getDependResultForItem(DependentItem item, Map<Long, TaskInstance> completeTaskList) {
DependResult dependResult = DependResult.SUCCESS;
if (!completeTaskList.containsKey(item.getDepTaskCode())) {
log.info("depend item: {} have not completed yet.", item.getDepTaskCode());
dependResult = DependResult.FAILED;
return dependResult;
}
TaskInstance taskInstance = completeTaskList.get(item.getDepTaskCode());
if (taskInstance.getState() != item.getStatus()) {
log.info("depend item : {} expect status: {}, actual status: {}", item.getDepTaskCode(), item.getStatus(),
taskInstance.getState().name());
dependResult = DependResult.FAILED;
}
log.info("Dependent item complete {} {},{}",
Constants.DEPENDENT_SPLIT, item.getDepTaskCode(), dependResult);
return dependResult;
}
}

51
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.runner.task.blocking;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class BlockingLogicTaskPluginFactory implements ILogicTaskPluginFactory<BlockingLogicTask> {
@Autowired
private ProcessInstanceDao processInstanceDao;
@Autowired
private TaskInstanceDao taskInstanceDao;
@Autowired
private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
@Override
public BlockingLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) {
return new BlockingLogicTask(taskExecutionContext, processInstanceExecCacheManager, processInstanceDao,
taskInstanceDao);
}
@Override
public String getTaskType() {
return BlockingLogicTask.TASK_TYPE;
}
}

51
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java

@ -17,9 +17,8 @@
package org.apache.dolphinscheduler.server.master.runner.task.condition; package org.apache.dolphinscheduler.server.master.runner.task.condition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
@ -27,6 +26,8 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask; import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
import java.util.List; import java.util.List;
@ -37,47 +38,42 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j @Slf4j
public class ConditionLogicTask extends BaseSyncLogicTask<ConditionsParameters> { public class ConditionLogicTask extends BaseSyncLogicTask<ConditionsParameters> {
public static final String TASK_TYPE = "CONDITIONS"; public static final String TASK_TYPE = "CONDITIONS";
private final TaskInstanceDao taskInstanceDao; private final TaskInstanceDao taskInstanceDao;
private final ProcessInstanceDao workflowInstanceDao;
private final TaskInstance taskInstance;
public ConditionLogicTask(TaskExecutionContext taskExecutionContext, public ConditionLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable,
TaskInstance taskInstance, TaskExecutionContext taskExecutionContext,
TaskInstanceDao taskInstanceDao, TaskInstanceDao taskInstanceDao) throws LogicTaskInitializeException {
ProcessInstanceDao workflowInstanceDao) { super(workflowExecuteRunnable, taskExecutionContext,
// todo: we need to change the parameter in front-end, so that we can directly use json to parse JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<ConditionsParameters>() {
super(taskExecutionContext, taskInstance.getConditionsParameters()); }));
this.taskInstanceDao = taskInstanceDao; this.taskInstanceDao = taskInstanceDao;
this.workflowInstanceDao = workflowInstanceDao;
this.taskInstance = taskInstance;
} }
@Override @Override
public void handle() { public void handle() {
// calculate the conditionResult
DependResult conditionResult = calculateConditionResult(); DependResult conditionResult = calculateConditionResult();
log.info("The condition result is {}", conditionResult); log.info("The condition result is {}", conditionResult);
taskParameters.setConditionSuccess(conditionResult == DependResult.SUCCESS); taskParameters.getConditionResult().setConditionSuccess(conditionResult == DependResult.SUCCESS);
taskInstance.setConditionsParameters(taskParameters); taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters));
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
} }
private DependResult calculateConditionResult() { private DependResult calculateConditionResult() {
final ProcessInstance processInstance = final List<TaskInstance> taskInstances = taskInstanceDao.queryValidTaskListByWorkflowInstanceId(
workflowInstanceDao.queryById(taskExecutionContext.getProcessInstanceId()); taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTestFlag());
final List<TaskInstance> taskInstances = final Map<Long, TaskInstance> taskInstanceMap = taskInstances.stream()
taskInstanceDao.queryValidTaskListByWorkflowInstanceId(processInstance.getId(), .collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity()));
processInstance.getTestFlag());
final Map<Long, TaskInstance> taskInstanceMap =
taskInstances.stream().collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity()));
List<DependResult> dependResults = taskParameters.getDependTaskList().stream() ConditionsParameters.ConditionDependency dependence = taskParameters.getDependence();
List<DependResult> dependResults = dependence.getDependTaskList()
.stream()
.map(dependentTaskModel -> DependentUtils.getDependResultForRelation( .map(dependentTaskModel -> DependentUtils.getDependResultForRelation(
dependentTaskModel.getRelation(), dependentTaskModel.getRelation(),
dependentTaskModel.getDependItemList() dependentTaskModel.getDependItemList()
@ -85,7 +81,7 @@ public class ConditionLogicTask extends BaseSyncLogicTask<ConditionsParameters>
.map(dependentItem -> getDependResultForItem(dependentItem, taskInstanceMap)) .map(dependentItem -> getDependResultForItem(dependentItem, taskInstanceMap))
.collect(Collectors.toList()))) .collect(Collectors.toList())))
.collect(Collectors.toList()); .collect(Collectors.toList());
return DependentUtils.getDependResultForRelation(taskParameters.getRelation(), dependResults); return DependentUtils.getDependResultForRelation(dependence.getRelation(), dependResults);
} }
private DependResult getDependResultForItem(DependentItem item, Map<Long, TaskInstance> taskInstanceMap) { private DependResult getDependResultForItem(DependentItem item, Map<Long, TaskInstance> taskInstanceMap) {
@ -96,8 +92,9 @@ public class ConditionLogicTask extends BaseSyncLogicTask<ConditionsParameters>
return DependResult.FAILED; return DependResult.FAILED;
} }
DependResult dependResult = DependResult dependResult = Objects.equals(item.getStatus(), taskInstance.getState())
Objects.equals(item.getStatus(), taskInstance.getState()) ? DependResult.SUCCESS : DependResult.FAILED; ? DependResult.SUCCESS
: DependResult.FAILED;
log.info("The depend item: {}", item); log.info("The depend item: {}", item);
log.info("Expect status: {}", item.getStatus()); log.info("Expect status: {}", item.getStatus());
log.info("Actual status: {}", taskInstance.getState()); log.info("Actual status: {}", taskInstance.getState());

21
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java

@ -17,12 +17,12 @@
package org.apache.dolphinscheduler.server.master.runner.task.condition; package org.apache.dolphinscheduler.server.master.runner.task.condition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -36,28 +36,19 @@ public class ConditionLogicTaskPluginFactory implements ILogicTaskPluginFactory<
@Autowired @Autowired
private TaskInstanceDao taskInstanceDao; private TaskInstanceDao taskInstanceDao;
@Autowired
private ProcessInstanceDao processInstanceDao;
@Autowired @Autowired
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
@Override @Override
public ConditionLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException { public ConditionLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
TaskInstance taskInstance = WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId(
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) taskExecutionContext.getProcessInstanceId());
.getTaskInstance(taskExecutionContext.getTaskInstanceId()) return new ConditionLogicTask(workflowExecuteRunnable, taskExecutionContext, taskInstanceDao);
.orElseThrow(() -> new LogicTaskInitializeException(
"Cannot find the task instance in workflow execute runnable"));
return new ConditionLogicTask(
taskExecutionContext,
taskInstance,
taskInstanceDao,
processInstanceDao);
} }
@Override @Override
public String getTaskType() { public String getTaskType() {
return ConditionLogicTask.TASK_TYPE; return ConditionsLogicTaskChannelFactory.NAME;
} }
} }

15
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java

@ -122,10 +122,12 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
private List<DependentExecute> initializeDependentTaskList() { private List<DependentExecute> initializeDependentTaskList() {
log.info("Begin to initialize dependent task list"); log.info("Begin to initialize dependent task list");
List<DependentTaskModel> dependTaskList = dependentParameters.getDependence().getDependTaskList();
final Set<Long> projectCodes = new HashSet<>(); final Set<Long> projectCodes = new HashSet<>();
final Set<Long> processDefinitionCodes = new HashSet<>(); final Set<Long> processDefinitionCodes = new HashSet<>();
final Set<Long> taskDefinitionCodes = new HashSet<>(); final Set<Long> taskDefinitionCodes = new HashSet<>();
for (DependentTaskModel taskModel : dependentParameters.getDependTaskList()) { for (DependentTaskModel taskModel : dependTaskList) {
for (DependentItem dependentItem : taskModel.getDependItemList()) { for (DependentItem dependentItem : taskModel.getDependItemList()) {
projectCodes.add(dependentItem.getProjectCode()); projectCodes.add(dependentItem.getProjectCode());
processDefinitionCodes.add(dependentItem.getDefinitionCode()); processDefinitionCodes.add(dependentItem.getDefinitionCode());
@ -142,7 +144,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
.collect(Collectors.toMap(TaskDefinition::getCode, Function.identity())); .collect(Collectors.toMap(TaskDefinition::getCode, Function.identity()));
final TaskInstance taskInstance = final TaskInstance taskInstance =
taskInstanceDao.queryById(taskExecutionContext.getTaskInstanceId()); taskInstanceDao.queryById(taskExecutionContext.getTaskInstanceId());
List<DependentExecute> dependentExecutes = dependentParameters.getDependTaskList() List<DependentExecute> dependentExecutes = dependTaskList
.stream() .stream()
.map(dependentTaskModel -> { .map(dependentTaskModel -> {
for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) {
@ -210,7 +212,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
} }
dependResultList.add(dependResult); dependResultList.add(dependResult);
} }
return DependentUtils.getDependResultForRelation(this.dependentParameters.getRelation(), return DependentUtils.getDependResultForRelation(dependentParameters.getDependence().getRelation(),
dependResultList); dependResultList);
} }
@ -218,7 +220,8 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
boolean isAllDependentTaskFinished = true; boolean isAllDependentTaskFinished = true;
for (DependentExecute dependentExecute : dependentTaskList) { for (DependentExecute dependentExecute : dependentTaskList) {
if (!dependentExecute.finish(dependentDate, processInstance.getTestFlag(), if (!dependentExecute.finish(dependentDate, processInstance.getTestFlag(),
dependentParameters.getFailurePolicy(), dependentParameters.getFailureWaitingTime())) { dependentParameters.getDependence().getFailurePolicy(),
dependentParameters.getDependence().getFailureWaitingTime())) {
isAllDependentTaskFinished = false; isAllDependentTaskFinished = false;
} }
dependentExecute.getDependResultMap().forEach((dependentKey, dependResult) -> { dependentExecute.getDependResultMap().forEach((dependentKey, dependResult) -> {
@ -238,7 +241,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
@Override @Override
public @NonNull Duration getAsyncTaskStateCheckInterval() { public @NonNull Duration getAsyncTaskStateCheckInterval() {
return dependentParameters.getCheckInterval() == null ? DEFAULT_STATE_CHECK_INTERVAL return dependentParameters.getDependence().getCheckInterval() == null ? DEFAULT_STATE_CHECK_INTERVAL
: Duration.ofSeconds(dependentParameters.getCheckInterval()); : Duration.ofSeconds(dependentParameters.getDependence().getCheckInterval());
} }
} }

20
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.master.runner.task.dependent; package org.apache.dolphinscheduler.server.master.runner.task.dependent;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao; import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
@ -26,8 +27,6 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction; import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
@ -37,6 +36,8 @@ import java.util.Date;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j @Slf4j
public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters> { public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters> {
@ -48,7 +49,7 @@ public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters>
private final TaskInstanceDao taskInstanceDao; private final TaskInstanceDao taskInstanceDao;
private final ProcessInstanceDao processInstanceDao; private final ProcessInstanceDao processInstanceDao;
private final ProcessInstanceExecCacheManager processInstanceExecCacheManager; private final WorkflowExecuteRunnable workflowExecuteRunnable;
public DependentLogicTask(TaskExecutionContext taskExecutionContext, public DependentLogicTask(TaskExecutionContext taskExecutionContext,
ProjectDao projectDao, ProjectDao projectDao,
@ -56,19 +57,16 @@ public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters>
TaskDefinitionDao taskDefinitionDao, TaskDefinitionDao taskDefinitionDao,
TaskInstanceDao taskInstanceDao, TaskInstanceDao taskInstanceDao,
ProcessInstanceDao processInstanceDao, ProcessInstanceDao processInstanceDao,
ProcessInstanceExecCacheManager processInstanceExecCacheManager) throws LogicTaskInitializeException { WorkflowExecuteRunnable workflowExecuteRunnable) {
super(taskExecutionContext, super(taskExecutionContext,
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<DependentParameters>() {
.getTaskInstance(taskExecutionContext.getTaskInstanceId()) }));
.orElseThrow(() -> new LogicTaskInitializeException(
"Cannot find the task instance in workflow execute runnable"))
.getDependency());
this.projectDao = projectDao; this.projectDao = projectDao;
this.processDefinitionDao = processDefinitionDao; this.processDefinitionDao = processDefinitionDao;
this.taskDefinitionDao = taskDefinitionDao; this.taskDefinitionDao = taskDefinitionDao;
this.taskInstanceDao = taskInstanceDao; this.taskInstanceDao = taskInstanceDao;
this.processInstanceDao = processInstanceDao; this.processInstanceDao = processInstanceDao;
this.processInstanceExecCacheManager = processInstanceExecCacheManager; this.workflowExecuteRunnable = workflowExecuteRunnable;
} }
@ -85,8 +83,6 @@ public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters>
@Override @Override
public void pause() throws MasterTaskExecuteException { public void pause() throws MasterTaskExecuteException {
WorkflowExecuteRunnable workflowExecuteRunnable =
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId());
if (workflowExecuteRunnable == null) { if (workflowExecuteRunnable == null) {
log.error("Cannot find the WorkflowExecuteRunnable"); log.error("Cannot find the WorkflowExecuteRunnable");
return; return;

9
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java

@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -52,6 +53,12 @@ public class DependentLogicTaskPluginFactory implements ILogicTaskPluginFactory<
@Override @Override
public DependentLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException { public DependentLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
int workflowInstanceId = taskExecutionContext.getProcessInstanceId();
WorkflowExecuteRunnable workflowExecuteRunnable =
processInstanceExecCacheManager.getByProcessInstanceId(workflowInstanceId);
if (workflowExecuteRunnable == null) {
throw new LogicTaskInitializeException("Cannot find the WorkflowExecuteRunnable: " + workflowInstanceId);
}
return new DependentLogicTask( return new DependentLogicTask(
taskExecutionContext, taskExecutionContext,
projectDao, projectDao,
@ -59,7 +66,7 @@ public class DependentLogicTaskPluginFactory implements ILogicTaskPluginFactory<
taskDefinitionDao, taskDefinitionDao,
taskInstanceDao, taskInstanceDao,
processInstanceDao, processInstanceDao,
processInstanceExecCacheManager); workflowExecuteRunnable);
} }
@Override @Override

13
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventLis
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction; import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
@ -41,29 +40,27 @@ import com.fasterxml.jackson.core.type.TypeReference;
public class SubWorkflowLogicTask extends BaseAsyncLogicTask<SubProcessParameters> { public class SubWorkflowLogicTask extends BaseAsyncLogicTask<SubProcessParameters> {
public static final String TASK_TYPE = "SUB_PROCESS"; public static final String TASK_TYPE = "SUB_PROCESS";
private final ProcessInstanceExecCacheManager processInstanceExecCacheManager; private final WorkflowExecuteRunnable workflowExecuteRunnable;
private final ProcessInstanceDao processInstanceDao; private final ProcessInstanceDao processInstanceDao;
public SubWorkflowLogicTask(TaskExecutionContext taskExecutionContext, public SubWorkflowLogicTask(TaskExecutionContext taskExecutionContext,
ProcessInstanceExecCacheManager processInstanceExecCacheManager, WorkflowExecuteRunnable workflowExecuteRunnable,
ProcessInstanceDao processInstanceDao) { ProcessInstanceDao processInstanceDao) {
super(taskExecutionContext, super(taskExecutionContext,
JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<SubProcessParameters>() { JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<SubProcessParameters>() {
})); }));
this.processInstanceExecCacheManager = processInstanceExecCacheManager; this.workflowExecuteRunnable = workflowExecuteRunnable;
this.processInstanceDao = processInstanceDao; this.processInstanceDao = processInstanceDao;
} }
@Override @Override
public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() throws MasterTaskExecuteException { public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() {
// todo: create sub workflow instance here? // todo: create sub workflow instance here?
return new SubWorkflowAsyncTaskExecuteFunction(taskExecutionContext, processInstanceDao); return new SubWorkflowAsyncTaskExecuteFunction(taskExecutionContext, processInstanceDao);
} }
@Override @Override
public void pause() throws MasterTaskExecuteException { public void pause() throws MasterTaskExecuteException {
WorkflowExecuteRunnable workflowExecuteRunnable =
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId());
if (workflowExecuteRunnable == null) { if (workflowExecuteRunnable == null) {
log.warn("Cannot find WorkflowExecuteRunnable"); log.warn("Cannot find WorkflowExecuteRunnable");
return; return;
@ -99,8 +96,6 @@ public class SubWorkflowLogicTask extends BaseAsyncLogicTask<SubProcessParameter
@Override @Override
public void kill() { public void kill() {
WorkflowExecuteRunnable workflowExecuteRunnable =
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId());
if (workflowExecuteRunnable == null) { if (workflowExecuteRunnable == null) {
log.warn("Cannot find WorkflowExecuteRunnable"); log.warn("Cannot find WorkflowExecuteRunnable");
return; return;

13
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskPluginFactory.java

@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.server.master.runner.task.subworkflow;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -37,8 +39,15 @@ public class SubWorkflowLogicTaskPluginFactory implements ILogicTaskPluginFactor
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
@Override @Override
public SubWorkflowLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) { public SubWorkflowLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
return new SubWorkflowLogicTask(taskExecutionContext, processInstanceExecCacheManager, processInstanceDao); int workflowInstanceId = taskExecutionContext.getProcessInstanceId();
WorkflowExecuteRunnable workflowExecuteRunnable =
processInstanceExecCacheManager.getByProcessInstanceId(workflowInstanceId);
if (workflowExecuteRunnable == null) {
throw new LogicTaskInitializeException(
"Cannot find the WorkflowExecuteRunnable by : " + workflowInstanceId);
}
return new SubWorkflowLogicTask(taskExecutionContext, workflowExecuteRunnable, processInstanceDao);
} }
@Override @Override

122
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java

@ -24,21 +24,26 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph;
import org.apache.dolphinscheduler.server.master.runner.IWorkflowExecuteContext;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask; import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
import org.apache.dolphinscheduler.server.master.utils.SwitchTaskUtils; import org.apache.dolphinscheduler.server.master.utils.SwitchTaskUtils;
import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j @Slf4j
public class SwitchLogicTask extends BaseSyncLogicTask<SwitchParameters> { public class SwitchLogicTask extends BaseSyncLogicTask<SwitchParameters> {
@ -47,121 +52,96 @@ public class SwitchLogicTask extends BaseSyncLogicTask<SwitchParameters> {
private final WorkflowExecuteRunnable workflowExecuteRunnable; private final WorkflowExecuteRunnable workflowExecuteRunnable;
private final TaskInstance taskInstance; private final TaskInstance taskInstance;
public SwitchLogicTask(TaskExecutionContext taskExecutionContext, public SwitchLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable,
ProcessInstanceExecCacheManager processInstanceExecCacheManager) throws LogicTaskInitializeException { TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
super(taskExecutionContext, super(workflowExecuteRunnable,
// todo: we need to refactor the logic task parameter........ taskExecutionContext,
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<SwitchParameters>() {
.getTaskInstance(taskExecutionContext.getTaskInstanceId()) }));
.orElseThrow(() -> new LogicTaskInitializeException( this.workflowExecuteRunnable = workflowExecuteRunnable;
"Cannot find the task instance in workflow execute runnable")) this.taskInstance =
.getSwitchDependency()); workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId()).orElseThrow(
this.workflowExecuteRunnable = () -> new LogicTaskInitializeException(
processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId());
this.taskInstance = workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId())
.orElseThrow(() -> new LogicTaskInitializeException(
"Cannot find the task instance in workflow execute runnable")); "Cannot find the task instance in workflow execute runnable"));
// Since the default branch is not in the dependTaskList, we need to add it to the end
// otherwise the default branch will never be skipped in DAGHelper
addDefaultBranchToEnd();
} }
@Override @Override
public void handle() throws MasterTaskExecuteException { public void handle() throws MasterTaskExecuteException {
if (CollectionUtils.isEmpty(taskParameters.getDependTaskList())) { if (CollectionUtils.isEmpty(taskParameters.getSwitchResult().getDependTaskList())) {
// If the branch is empty then will go into the default branch // If the branch is empty then will go into the default branch
// This case shouldn't happen, we can directly throw exception and forbid the user to set branch // This case shouldn't happen, we can directly throw exception and forbid the user to set branch
log.info("The switch items is empty");
moveToDefaultBranch(); moveToDefaultBranch();
} else { } else {
calculateSwitchBranch(); calculateSwitchBranch();
} }
taskInstance.setSwitchDependency(taskParameters); checkIfBranchExist(taskParameters.getNextBranch());
taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters));
taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
log.info("Switch task execute finished: {}", taskExecutionContext.getCurrentExecutionStatus().name()); log.info("Switch task execute finished: {}", taskExecutionContext.getCurrentExecutionStatus().name());
} }
private void moveToDefaultBranch() { private void moveToDefaultBranch() {
List<SwitchResultVo> switchResultVos = taskParameters.getDependTaskList(); log.info("Begin to move to the default branch");
SwitchResultVo defaultSwitchResultVo = getDefaultSwitchResultVo(); if (taskParameters.getSwitchResult().getNextNode() == null) {
checkIfBranchExist(defaultSwitchResultVo.getNextNode()); throw new IllegalArgumentException(
"The default branch is empty, please check the switch task configuration");
taskParameters.setResultConditionLocation(switchResultVos.size() - 1); }
taskParameters.setNextBranch(taskParameters.getSwitchResult().getNextNode());
log.info("The condition is not satisfied, move to the default branch: {}", log.info("The condition is not satisfied, move to the default branch: {}",
taskParameters.getNextNode().stream().map(node -> workflowExecuteRunnable.getWorkflowExecuteContext() getTaskName(taskParameters.getNextBranch()));
.getWorkflowGraph().getDag().getNode(node).getName()).collect(Collectors.toList()));
} }
private void calculateSwitchBranch() { private void calculateSwitchBranch() {
List<SwitchResultVo> switchResultVos = taskParameters.getDependTaskList(); List<SwitchResultVo> switchResultVos = taskParameters.getSwitchResult().getDependTaskList();
Map<String, Property> globalParams = taskExecutionContext.getPrepareParamsMap(); Map<String, Property> globalParams = taskExecutionContext.getPrepareParamsMap();
Map<String, Property> varParams = JSONUtils Map<String, Property> varParams = JSONUtils
.toList(taskInstance.getVarPool(), Property.class) .toList(taskInstance.getVarPool(), Property.class)
.stream() .stream()
.collect(Collectors.toMap(Property::getProp, Property -> Property)); .collect(Collectors.toMap(Property::getProp, Property -> Property));
int finalConditionLocation = -1; Long nextBranch = null;
// The last one is the default branch, no need to calculate for (SwitchResultVo switchResultVo : switchResultVos) {
for (int i = 0; i < switchResultVos.size() - 1; i++) { log.info("Begin to execute switch item: {} ", switchResultVo);
SwitchResultVo switchResultVo = switchResultVos.get(i);
log.info("Begin to execute {} condition: {} ", i, switchResultVo.getCondition());
String content = SwitchTaskUtils.generateContentWithTaskParams(switchResultVo.getCondition(), globalParams,
varParams);
log.info("Format condition sentence::{} successfully", content);
boolean result;
try { try {
result = SwitchTaskUtils.evaluate(content); String content = SwitchTaskUtils.generateContentWithTaskParams(switchResultVo.getCondition(),
log.info("Execute condition sentence: {} successfully: {}", content, result); globalParams, varParams);
if (result) { log.info("Format condition sentence::{} successfully", content);
boolean conditionResult = SwitchTaskUtils.evaluate(content);
log.info("Execute condition sentence: {} successfully: {}", content, conditionResult);
if (conditionResult) {
// If matched, break the loop // If matched, break the loop
finalConditionLocation = i; nextBranch = switchResultVo.getNextNode();
break; break;
} }
} catch (Exception e) { } catch (Exception e) {
log.info("Execute condition sentence: {} failed", content, e); log.info("Execute switch item: {} failed", switchResultVo, e);
} }
} }
// If the finalConditionLocation is -1, then the default branch will be executed
if (finalConditionLocation >= 0) { if (nextBranch == null) {
List<Long> nextNodes = switchResultVos.get(finalConditionLocation).getNextNode(); log.info("All switch item is not satisfied");
checkIfBranchExist(nextNodes);
log.info("The condition is satisfied, move to the branch: {}",
switchResultVos.get(finalConditionLocation).getNextNode().stream()
.map(node -> workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowGraph().getDag()
.getNode(node).getName())
.collect(Collectors.toList()));
taskParameters.setResultConditionLocation(finalConditionLocation);
} else {
moveToDefaultBranch(); moveToDefaultBranch();
} }
} }
private void checkIfBranchExist(List<Long> branchNode) { private void checkIfBranchExist(Long branchNode) {
if (CollectionUtils.isEmpty(branchNode)) { if (branchNode == null) {
throw new IllegalArgumentException("The branchNode is empty, please check the switch task configuration");
}
for (Long branch : branchNode) {
if (branch == null) {
throw new IllegalArgumentException("The branch is empty, please check the switch task configuration"); throw new IllegalArgumentException("The branch is empty, please check the switch task configuration");
} }
if (!workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowGraph().getDag().containsNode(branch)) { if (!workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowGraph().getDag().containsNode(branchNode)) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"The branch(code= " + branchNode "The branch(code= " + branchNode
+ ") is not in the dag, please check the switch task configuration"); + ") is not in the dag, please check the switch task configuration");
} }
} }
}
private void addDefaultBranchToEnd() {
SwitchResultVo switchResultVo = new SwitchResultVo(null, taskParameters.getNextNode());
List<SwitchResultVo> dependTaskList = taskParameters.getDependTaskList();
if (!dependTaskList.contains(switchResultVo)) {
dependTaskList.add(switchResultVo);
}
}
private SwitchResultVo getDefaultSwitchResultVo() { private String getTaskName(Long taskCode) {
return taskParameters.getDependTaskList().get(taskParameters.getDependTaskList().size() - 1); Optional<TaskNode> taskNode = Optional.ofNullable(workflowExecuteRunnable.getWorkflowExecuteContext())
.map(IWorkflowExecuteContext::getWorkflowGraph)
.map(IWorkflowGraph::getDag)
.map(dag -> dag.getNode(taskCode));
return taskNode.map(TaskNode::getName).orElse(null);
} }
} }

10
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.runner.task.switchtask;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -36,7 +37,14 @@ public class SwitchLogicTaskPluginFactory implements ILogicTaskPluginFactory<Swi
@Override @Override
public SwitchLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException { public SwitchLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
return new SwitchLogicTask(taskExecutionContext, processInstanceExecCacheManager); int workflowInstanceId = taskExecutionContext.getProcessInstanceId();
WorkflowExecuteRunnable workflowExecuteRunnable =
processInstanceExecCacheManager.getByProcessInstanceId(workflowInstanceId);
if (workflowExecuteRunnable == null) {
throw new LogicTaskInitializeException(
"Cannot find the WorkflowExecuteRunnable by : " + workflowInstanceId);
}
return new SwitchLogicTask(workflowExecuteRunnable, taskExecutionContext);
} }
@Override @Override

4
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java

@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdResponse;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder;
@ -40,7 +41,6 @@ import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.utils.ProcessUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils;
@ -167,7 +167,7 @@ public class WorkerFailoverService {
taskInstance.setProcessInstance(processInstance); taskInstance.setProcessInstance(processInstance);
if (!TaskUtils.isMasterTask(taskInstance.getTaskType())) { if (!TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) {
killYarnTask(taskInstance, processInstance); killYarnTask(taskInstance, processInstance);
} else { } else {
log.info("The failover taskInstance is a master task, no need to failover in worker failover"); log.info("The failover taskInstance is a master task, no need to failover in worker failover");

48
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/TaskUtils.java

@ -1,48 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.utils;
import org.apache.dolphinscheduler.server.master.runner.task.blocking.BlockingLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.condition.ConditionLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.dependent.DependentLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.dynamic.DynamicLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.subworkflow.SubWorkflowLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.switchtask.SwitchLogicTask;
import java.util.Set;
import lombok.experimental.UtilityClass;
import com.google.common.collect.Sets;
@UtilityClass
public class TaskUtils {
// todo: Add to SPI
private final Set<String> MASTER_TASK_TYPES = Sets.newHashSet(
BlockingLogicTask.TASK_TYPE,
ConditionLogicTask.TASK_TYPE,
DependentLogicTask.TASK_TYPE,
SubWorkflowLogicTask.TASK_TYPE,
SwitchLogicTask.TASK_TYPE,
DynamicLogicTask.TASK_TYPE);
public boolean isMasterTask(String taskType) {
return MASTER_TASK_TYPES.contains(taskType);
}
}

1
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java

@ -392,7 +392,6 @@ public class WorkflowExecuteRunnableTest {
// task instance already finished, not dispatch // task instance already finished, not dispatch
TaskInstance taskInstance = new TaskInstance(); TaskInstance taskInstance = new TaskInstance();
taskInstance.setState(TaskExecutionStatus.PAUSE); taskInstance.setState(TaskExecutionStatus.PAUSE);
Mockito.when(processInstance.isBlocked()).thenReturn(true);
TaskExecuteRunnable taskExecuteRunnable = Mockito.mock(TaskExecuteRunnable.class); TaskExecuteRunnable taskExecuteRunnable = Mockito.mock(TaskExecuteRunnable.class);
workflowExecuteThread.tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable); workflowExecuteThread.tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable);
Mockito.verify(taskExecuteRunnable, Mockito.never()).dispatch(); Mockito.verify(taskExecuteRunnable, Mockito.never()).dispatch();

47
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactoryTest.java

@ -17,15 +17,18 @@
package org.apache.dolphinscheduler.server.master.runner.dispatcher; package org.apache.dolphinscheduler.server.master.runner.dispatcher;
import org.apache.dolphinscheduler.server.master.runner.task.blocking.BlockingLogicTask; import static com.google.common.truth.Truth.assertThat;
import org.apache.dolphinscheduler.server.master.runner.task.condition.ConditionLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.dependent.DependentLogicTask; import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.runner.task.subworkflow.SubWorkflowLogicTask; import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.runner.task.switchtask.SwitchLogicTask; import org.apache.dolphinscheduler.plugin.task.api.task.DynamicLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory;
import org.junit.jupiter.api.Assertions; import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import org.junit.jupiter.api.Test; import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
@ -42,18 +45,20 @@ public class TaskDispatchFactoryTest {
@Mock @Mock
private WorkerTaskDispatcher workerTaskDispatcher; private WorkerTaskDispatcher workerTaskDispatcher;
@Test @ParameterizedTest
public void getTaskDispatcher() { @ValueSource(strings = {
Assertions.assertEquals(masterTaskDispatcher, ConditionsLogicTaskChannelFactory.NAME,
taskDispatchFactory.getTaskDispatcher(BlockingLogicTask.TASK_TYPE)); DependentLogicTaskChannelFactory.NAME,
Assertions.assertEquals(masterTaskDispatcher, DynamicLogicTaskChannelFactory.NAME,
taskDispatchFactory.getTaskDispatcher(ConditionLogicTask.TASK_TYPE)); SubWorkflowLogicTaskChannelFactory.NAME,
Assertions.assertEquals(masterTaskDispatcher, SwitchLogicTaskChannelFactory.NAME})
taskDispatchFactory.getTaskDispatcher(DependentLogicTask.TASK_TYPE)); public void getTaskDispatcher_withLogicTask(String taskType) {
Assertions.assertEquals(masterTaskDispatcher, assertThat(taskDispatchFactory.getTaskDispatcher(taskType)).isSameInstanceAs(masterTaskDispatcher);
taskDispatchFactory.getTaskDispatcher(SubWorkflowLogicTask.TASK_TYPE)); }
Assertions.assertEquals(masterTaskDispatcher, taskDispatchFactory.getTaskDispatcher(SwitchLogicTask.TASK_TYPE));
@ParameterizedTest
Assertions.assertEquals(workerTaskDispatcher, taskDispatchFactory.getTaskDispatcher("SHELL")); @ValueSource(strings = {ShellTaskChannelFactory.NAME})
public void getTaskDispatcher_withWorkerTask(String taskType) {
assertThat(taskDispatchFactory.getTaskDispatcher(taskType)).isSameInstanceAs(workerTaskDispatcher);
} }
} }

10
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.master.runner.task.dynamic;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
@ -43,8 +44,6 @@ import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
class DynamicLogicTaskTest { class DynamicLogicTaskTest {
@ -74,14 +73,11 @@ class DynamicLogicTaskTest {
private DynamicLogicTask dynamicLogicTask; private DynamicLogicTask dynamicLogicTask;
private ObjectMapper objectMapper;
@BeforeEach @BeforeEach
public void setUp() { public void setUp() {
// Set up your test environment before each test. // Set up your test environment before each test.
dynamicParameters = new DynamicParameters(); dynamicParameters = new DynamicParameters();
taskExecutionContext = Mockito.mock(TaskExecutionContext.class); taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
objectMapper = new ObjectMapper();
processInstance = new ProcessInstance(); processInstance = new ProcessInstance();
Mockito.when(processInstanceDao.queryById(Mockito.any())).thenReturn(processInstance); Mockito.when(processInstanceDao.queryById(Mockito.any())).thenReturn(processInstance);
dynamicLogicTask = new DynamicLogicTask( dynamicLogicTask = new DynamicLogicTask(
@ -95,7 +91,7 @@ class DynamicLogicTaskTest {
} }
@Test @Test
void testGenerateParameterGroup() throws Exception { void testGenerateParameterGroup() {
DynamicInputParameter dynamicInputParameter1 = new DynamicInputParameter(); DynamicInputParameter dynamicInputParameter1 = new DynamicInputParameter();
dynamicInputParameter1.setName("param1"); dynamicInputParameter1.setName("param1");
dynamicInputParameter1.setValue("a,b,c"); dynamicInputParameter1.setValue("a,b,c");
@ -113,7 +109,7 @@ class DynamicLogicTaskTest {
Mockito.when(taskExecutionContext.getPrepareParamsMap()).thenReturn(new HashMap<>()); Mockito.when(taskExecutionContext.getPrepareParamsMap()).thenReturn(new HashMap<>());
Mockito.when(taskExecutionContext.getTaskParams()) Mockito.when(taskExecutionContext.getTaskParams())
.thenReturn(objectMapper.writeValueAsString(dynamicParameters)); .thenReturn(JSONUtils.toJsonString(dynamicParameters));
dynamicLogicTask = new DynamicLogicTask( dynamicLogicTask = new DynamicLogicTask(
taskExecutionContext, taskExecutionContext,

8
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.server.master.service; package org.apache.dolphinscheduler.server.master.service;
import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_TASK_TYPE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doNothing;
@ -31,6 +29,8 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory;
import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
@ -139,13 +139,13 @@ public class FailoverServiceTest {
masterTaskInstance.setId(1); masterTaskInstance.setId(1);
masterTaskInstance.setStartTime(new Date()); masterTaskInstance.setStartTime(new Date());
masterTaskInstance.setHost(testMasterHost); masterTaskInstance.setHost(testMasterHost);
masterTaskInstance.setTaskType(TASK_TYPE_SWITCH); masterTaskInstance.setTaskType(SwitchLogicTaskChannelFactory.NAME);
workerTaskInstance = new TaskInstance(); workerTaskInstance = new TaskInstance();
workerTaskInstance.setId(2); workerTaskInstance.setId(2);
workerTaskInstance.setStartTime(new Date()); workerTaskInstance.setStartTime(new Date());
workerTaskInstance.setHost(testWorkerHost); workerTaskInstance.setHost(testWorkerHost);
workerTaskInstance.setTaskType(COMMON_TASK_TYPE); workerTaskInstance.setTaskType(ShellTaskChannelFactory.NAME);
given(processService.queryNeedFailoverProcessInstances(Mockito.anyString())) given(processService.queryNeedFailoverProcessInstances(Mockito.anyString()))
.willReturn(Arrays.asList(processInstance)); .willReturn(Arrays.asList(processInstance));

39
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/TaskUtilsTest.java

@ -1,39 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.utils;
import org.apache.dolphinscheduler.server.master.runner.task.blocking.BlockingLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.condition.ConditionLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.dependent.DependentLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.subworkflow.SubWorkflowLogicTask;
import org.apache.dolphinscheduler.server.master.runner.task.switchtask.SwitchLogicTask;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class TaskUtilsTest {
@Test
public void isMasterTask() {
Assertions.assertTrue(TaskUtils.isMasterTask(BlockingLogicTask.TASK_TYPE));
Assertions.assertTrue(TaskUtils.isMasterTask(ConditionLogicTask.TASK_TYPE));
Assertions.assertTrue(TaskUtils.isMasterTask(DependentLogicTask.TASK_TYPE));
Assertions.assertTrue(TaskUtils.isMasterTask(SubWorkflowLogicTask.TASK_TYPE));
Assertions.assertTrue(TaskUtils.isMasterTask(SwitchLogicTask.TASK_TYPE));
}
}

129
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java

@ -17,27 +17,17 @@
package org.apache.dolphinscheduler.service.model; package org.apache.dolphinscheduler.service.model;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.enums.TaskExecuteType;
import org.apache.dolphinscheduler.common.model.PreviousTaskNode;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Objects; import java.util.Objects;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@ -118,11 +108,6 @@ public class TaskNode {
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class) @JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
private String preTasks; private String preTasks;
/**
* node dependency list
*/
private List<PreviousTaskNode> preTaskNodeList;
/** /**
* users store additional information * users store additional information
*/ */
@ -135,25 +120,6 @@ public class TaskNode {
*/ */
private List<Long> depList; private List<Long> depList;
/**
* outer dependency information
*/
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class)
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
private String dependence;
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class)
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
private String conditionResult;
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class)
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
private String switchResult;
@JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class)
@JsonSerialize(using = JSONUtils.JsonDataSerializer.class)
private String waitStartTimeout;
/** /**
* task instance priority * task instance priority
*/ */
@ -312,10 +278,8 @@ public class TaskNode {
&& Objects.equals(preTasks, taskNode.preTasks) && Objects.equals(preTasks, taskNode.preTasks)
&& Objects.equals(extras, taskNode.extras) && Objects.equals(extras, taskNode.extras)
&& Objects.equals(runFlag, taskNode.runFlag) && Objects.equals(runFlag, taskNode.runFlag)
&& Objects.equals(dependence, taskNode.dependence)
&& Objects.equals(workerGroup, taskNode.workerGroup) && Objects.equals(workerGroup, taskNode.workerGroup)
&& Objects.equals(environmentCode, taskNode.environmentCode) && Objects.equals(environmentCode, taskNode.environmentCode)
&& Objects.equals(conditionResult, taskNode.conditionResult)
&& CollectionUtils.isEqualCollection(depList, taskNode.depList) && CollectionUtils.isEqualCollection(depList, taskNode.depList)
&& Objects.equals(taskExecuteType, taskNode.taskExecuteType); && Objects.equals(taskExecuteType, taskNode.taskExecuteType);
} }
@ -325,14 +289,6 @@ public class TaskNode {
return Objects.hash(name, desc, type, params, preTasks, extras, depList, runFlag); return Objects.hash(name, desc, type, params, preTasks, extras, depList, runFlag);
} }
public String getDependence() {
return dependence;
}
public void setDependence(String dependence) {
this.dependence = dependence;
}
public int getMaxRetryTimes() { public int getMaxRetryTimes() {
return maxRetryTimes; return maxRetryTimes;
} }
@ -373,14 +329,6 @@ public class TaskNode {
this.workerGroup = workerGroup; this.workerGroup = workerGroup;
} }
public String getConditionResult() {
return conditionResult;
}
public void setConditionResult(String conditionResult) {
this.conditionResult = conditionResult;
}
public int getDelayTime() { public int getDelayTime() {
return delayTime; return delayTime;
} }
@ -405,64 +353,6 @@ public class TaskNode {
this.version = version; this.version = version;
} }
/**
* get task time out parameter
*
* @return task time out parameter
*/
public TaskTimeoutParameter getTaskTimeoutParameter() {
if (!StringUtils.isEmpty(this.getTimeout())) {
String formatStr =
String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name());
return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class);
}
return new TaskTimeoutParameter(false);
}
public boolean isConditionsTask() {
return TASK_TYPE_CONDITIONS.equalsIgnoreCase(this.getType());
}
public boolean isSwitchTask() {
return TASK_TYPE_SWITCH.equalsIgnoreCase(this.getType());
}
public List<PreviousTaskNode> getPreTaskNodeList() {
return preTaskNodeList;
}
public boolean isBlockingTask() {
return TASK_TYPE_BLOCKING.equalsIgnoreCase(this.getType());
}
public void setPreTaskNodeList(List<PreviousTaskNode> preTaskNodeList) {
this.preTaskNodeList = preTaskNodeList;
}
public String getTaskParams() {
Map<String, Object> taskParams = JSONUtils.parseObject(this.params, new TypeReference<Map<String, Object>>() {
});
if (taskParams == null) {
taskParams = new HashMap<>();
}
taskParams.put(Constants.CONDITION_RESULT, this.conditionResult);
taskParams.put(Constants.DEPENDENCE, this.dependence);
taskParams.put(Constants.SWITCH_RESULT, this.switchResult);
taskParams.put(Constants.WAIT_START_TIMEOUT, this.waitStartTimeout);
return JSONUtils.toJsonString(taskParams);
}
public Map<String, Object> taskParamsToJsonObj(String taskParams) {
Map<String, Object> taskParamsMap = JSONUtils.parseObject(taskParams, new TypeReference<Map<String, Object>>() {
});
if (taskParamsMap == null) {
taskParamsMap = new HashMap<>();
}
return taskParamsMap;
}
@Override @Override
public String toString() { public String toString() {
return "TaskNode{" return "TaskNode{"
@ -478,11 +368,8 @@ public class TaskNode {
+ ", retryInterval=" + retryInterval + ", retryInterval=" + retryInterval
+ ", params='" + params + '\'' + ", params='" + params + '\''
+ ", preTasks='" + preTasks + '\'' + ", preTasks='" + preTasks + '\''
+ ", preTaskNodeList=" + preTaskNodeList
+ ", extras='" + extras + '\'' + ", extras='" + extras + '\''
+ ", depList=" + depList + ", depList=" + depList
+ ", dependence='" + dependence + '\''
+ ", conditionResult='" + conditionResult + '\''
+ ", taskInstancePriority=" + taskInstancePriority + ", taskInstancePriority=" + taskInstancePriority
+ ", workerGroup='" + workerGroup + '\'' + ", workerGroup='" + workerGroup + '\''
+ ", environmentCode=" + environmentCode + ", environmentCode=" + environmentCode
@ -500,22 +387,6 @@ public class TaskNode {
return this.environmentCode; return this.environmentCode;
} }
public String getSwitchResult() {
return switchResult;
}
public void setSwitchResult(String switchResult) {
this.switchResult = switchResult;
}
public String getWaitStartTimeout() {
return this.waitStartTimeout;
}
public void setWaitStartTimeout(String waitStartTimeout) {
this.waitStartTimeout = waitStartTimeout;
}
public int getTaskGroupId() { public int getTaskGroupId() {
return taskGroupId; return taskGroupId;
} }

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -176,8 +176,6 @@ public interface ProcessService {
DqComparisonType getComparisonTypeById(int id); DqComparisonType getComparisonTypeById(int id);
void changeTaskGroupQueueStatus(int taskId, TaskGroupQueueStatus status);
TaskGroupQueue insertIntoTaskGroupQueue(Integer taskId, TaskGroupQueue insertIntoTaskGroupQueue(Integer taskId,
String taskName, String taskName,
Integer groupId, Integer groupId,

29
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java

@ -120,6 +120,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.cron.CronUtils;
import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.exceptions.CronParseException;
@ -1051,7 +1052,7 @@ public class ProcessServiceImpl implements ProcessService {
*/ */
private void initTaskInstance(TaskInstance taskInstance) { private void initTaskInstance(TaskInstance taskInstance) {
if (!taskInstance.isSubProcess() if (!TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())
&& (taskInstance.getState().isKill() || taskInstance.getState().isFailure())) { && (taskInstance.getState().isKill() || taskInstance.getState().isFailure())) {
taskInstance.setFlag(Flag.NO); taskInstance.setFlag(Flag.NO);
taskInstanceDao.updateById(taskInstance); taskInstanceDao.updateById(taskInstance);
@ -1169,7 +1170,7 @@ public class ProcessServiceImpl implements ProcessService {
*/ */
@Override @Override
public void createSubWorkProcess(ProcessInstance parentProcessInstance, TaskInstance task) { public void createSubWorkProcess(ProcessInstance parentProcessInstance, TaskInstance task) {
if (!task.isSubProcess()) { if (!TaskTypeUtils.isSubWorkflowTask(task.getTaskType())) {
return; return;
} }
// check create sub work flow firstly // check create sub work flow firstly
@ -1940,13 +1941,7 @@ public class ProcessServiceImpl implements ProcessService {
: Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); : Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
taskNode.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes()); taskNode.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes());
taskNode.setRetryInterval(taskDefinitionLog.getFailRetryInterval()); taskNode.setRetryInterval(taskDefinitionLog.getFailRetryInterval());
Map<String, Object> taskParamsMap = taskNode.taskParamsToJsonObj(taskDefinitionLog.getTaskParams()); taskNode.setParams(taskDefinitionLog.getTaskParams());
taskNode.setConditionResult(JSONUtils.toJsonString(taskParamsMap.get(Constants.CONDITION_RESULT)));
taskNode.setSwitchResult(JSONUtils.toJsonString(taskParamsMap.get(Constants.SWITCH_RESULT)));
taskNode.setDependence(JSONUtils.toJsonString(taskParamsMap.get(Constants.DEPENDENCE)));
taskParamsMap.remove(Constants.CONDITION_RESULT);
taskParamsMap.remove(Constants.DEPENDENCE);
taskNode.setParams(JSONUtils.toJsonString(taskParamsMap));
taskNode.setTaskInstancePriority(taskDefinitionLog.getTaskPriority()); taskNode.setTaskInstancePriority(taskDefinitionLog.getTaskPriority());
taskNode.setWorkerGroup(taskDefinitionLog.getWorkerGroup()); taskNode.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
taskNode.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode()); taskNode.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode());
@ -2039,22 +2034,6 @@ public class ProcessServiceImpl implements ProcessService {
return dqComparisonTypeMapper.selectById(id); return dqComparisonTypeMapper.selectById(id);
} }
/**
* release the TGQ resource when the corresponding task is finished.
*
* @param taskId task id
* @return the result code and msg
*/
@Override
public void changeTaskGroupQueueStatus(int taskId, TaskGroupQueueStatus status) {
TaskGroupQueue taskGroupQueue = taskGroupQueueMapper.queryByTaskId(taskId);
taskGroupQueue.setInQueue(Flag.NO.getCode());
taskGroupQueue.setStatus(status);
taskGroupQueue.setUpdateTime(new Date(System.currentTimeMillis()));
taskGroupQueueMapper.updateById(taskGroupQueue);
}
@Override @Override
public TaskGroupQueue insertIntoTaskGroupQueue(Integer taskInstanceId, public TaskGroupQueue insertIntoTaskGroupQueue(Integer taskInstanceId,
String taskName, String taskName,

155
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java

@ -23,10 +23,12 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessDag; import org.apache.dolphinscheduler.service.process.ProcessDag;
@ -35,6 +37,7 @@ import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -44,6 +47,9 @@ import java.util.Set;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Lists;
/** /**
* dag tools * dag tools
*/ */
@ -228,22 +234,6 @@ public class DagHelper {
return processDag; return processDag;
} }
/**
* find node by node name
*
* @param nodeDetails nodeDetails
* @param nodeName nodeName
* @return task node
*/
public static TaskNode findNodeByName(List<TaskNode> nodeDetails, String nodeName) {
for (TaskNode taskNode : nodeDetails) {
if (taskNode.getName().equals(nodeName)) {
return taskNode;
}
}
return null;
}
/** /**
* find node by node code * find node by node code
* *
@ -305,10 +295,10 @@ public class DagHelper {
if (preNodeCode == null) { if (preNodeCode == null) {
startVertexes = dag.getBeginNode(); startVertexes = dag.getBeginNode();
} else if (dag.getNode(preNodeCode).isConditionsTask()) { } else if (TaskTypeUtils.isConditionTask(dag.getNode(preNodeCode).getType())) {
List<Long> conditionTaskList = parseConditionTask(preNodeCode, skipTaskNodeList, dag, completeTaskList); List<Long> conditionTaskList = parseConditionTask(preNodeCode, skipTaskNodeList, dag, completeTaskList);
startVertexes.addAll(conditionTaskList); startVertexes.addAll(conditionTaskList);
} else if (dag.getNode(preNodeCode).isSwitchTask()) { } else if (TaskTypeUtils.isSwitchTask(dag.getNode(preNodeCode).getType())) {
List<Long> conditionTaskList = parseSwitchTask(preNodeCode, skipTaskNodeList, dag, completeTaskList); List<Long> conditionTaskList = parseSwitchTask(preNodeCode, skipTaskNodeList, dag, completeTaskList);
startVertexes.addAll(conditionTaskList); startVertexes.addAll(conditionTaskList);
} else { } else {
@ -321,7 +311,7 @@ public class DagHelper {
continue; continue;
} }
if (isTaskNodeNeedSkip(taskNode, skipTaskNodeList)) { if (isTaskNodeNeedSkip(taskNode, skipTaskNodeList)) {
setTaskNodeSkip(subsequent, dag, completeTaskList, skipTaskNodeList); setTaskNodeSkip(subsequent, dag, skipTaskNodeList);
continue; continue;
} }
if (!DagHelper.allDependsForbiddenOrEnd(taskNode, dag, skipTaskNodeList, completeTaskList)) { if (!DagHelper.allDependsForbiddenOrEnd(taskNode, dag, skipTaskNodeList, completeTaskList)) {
@ -362,18 +352,20 @@ public class DagHelper {
Map<Long, TaskInstance> completeTaskList) { Map<Long, TaskInstance> completeTaskList) {
List<Long> conditionTaskList = new ArrayList<>(); List<Long> conditionTaskList = new ArrayList<>();
TaskNode taskNode = dag.getNode(nodeCode); TaskNode taskNode = dag.getNode(nodeCode);
if (!taskNode.isConditionsTask()) { if (!TaskTypeUtils.isConditionTask(taskNode.getType())) {
return conditionTaskList; return conditionTaskList;
} }
if (!completeTaskList.containsKey(nodeCode)) { if (!completeTaskList.containsKey(nodeCode)) {
return conditionTaskList; return conditionTaskList;
} }
TaskInstance taskInstance = completeTaskList.get(nodeCode); TaskInstance taskInstance = completeTaskList.get(nodeCode);
ConditionsParameters conditionsParameters = taskInstance.getConditionsParameters(); ConditionsParameters conditionsParameters =
ConditionsParameters.ConditionResult conditionResult = taskInstance.getConditionResult(); JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference<ConditionsParameters>() {
});
ConditionsParameters.ConditionResult conditionResult = conditionsParameters.getConditionResult();
List<Long> skipNodeList = new ArrayList<>(); List<Long> skipNodeList = new ArrayList<>();
if (conditionsParameters.isConditionSuccess()) { if (conditionResult.isConditionSuccess()) {
conditionTaskList = conditionResult.getSuccessNode(); conditionTaskList = conditionResult.getSuccessNode();
skipNodeList = conditionResult.getFailedNode(); skipNodeList = conditionResult.getFailedNode();
} else { } else {
@ -382,7 +374,7 @@ public class DagHelper {
} }
if (CollectionUtils.isNotEmpty(skipNodeList)) { if (CollectionUtils.isNotEmpty(skipNodeList)) {
skipNodeList.forEach(skipNode -> setTaskNodeSkip(skipNode, dag, completeTaskList, skipTaskNodeList)); skipNodeList.forEach(skipNode -> setTaskNodeSkip(skipNode, dag, skipTaskNodeList));
} }
// the conditionTaskList maybe null if no next task // the conditionTaskList maybe null if no next task
conditionTaskList = Optional.ofNullable(conditionTaskList).orElse(new ArrayList<>()); conditionTaskList = Optional.ofNullable(conditionTaskList).orElse(new ArrayList<>());
@ -402,82 +394,50 @@ public class DagHelper {
Map<Long, TaskInstance> completeTaskList) { Map<Long, TaskInstance> completeTaskList) {
List<Long> conditionTaskList = new ArrayList<>(); List<Long> conditionTaskList = new ArrayList<>();
TaskNode taskNode = dag.getNode(nodeCode); TaskNode taskNode = dag.getNode(nodeCode);
if (!taskNode.isSwitchTask()) { if (!SwitchLogicTaskChannelFactory.NAME.equals(taskNode.getType())) {
return conditionTaskList; return conditionTaskList;
} }
if (!completeTaskList.containsKey(nodeCode)) { if (!completeTaskList.containsKey(nodeCode)) {
return conditionTaskList; return conditionTaskList;
} }
conditionTaskList = skipTaskNode4Switch(taskNode, skipTaskNodeList, completeTaskList, dag); conditionTaskList = skipTaskNode4Switch(skipTaskNodeList, completeTaskList.get(nodeCode), dag);
return conditionTaskList; return conditionTaskList;
} }
public static List<Long> skipTaskNode4Switch(TaskNode taskNode, public static List<Long> skipTaskNode4Switch(Map<Long, TaskNode> skipTaskNodeList,
Map<Long, TaskNode> skipTaskNodeList, TaskInstance taskInstance,
Map<Long, TaskInstance> completeTaskList,
DAG<Long, TaskNode, TaskNodeRelation> dag) { DAG<Long, TaskNode, TaskNodeRelation> dag) {
SwitchParameters switchParameters = SwitchParameters switchParameters =
completeTaskList.get(taskNode.getCode()).getSwitchDependency(); JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference<SwitchParameters>() {
int resultConditionLocation = switchParameters.getResultConditionLocation(); });
List<SwitchResultVo> conditionResultVoList = switchParameters.getDependTaskList();
List<Long> switchTaskList = conditionResultVoList.get(resultConditionLocation).getNextNode(); SwitchParameters.SwitchResult switchResult = switchParameters.getSwitchResult();
Set<Long> switchNeedWorkCodes = new HashSet<>(); Long nextBranch = switchParameters.getNextBranch();
if (CollectionUtils.isEmpty(switchTaskList)) { if (switchResult == null) {
return new ArrayList<>(); log.error("switchResult is null, please check the switch task configuration");
} return Collections.emptyList();
// get all downstream nodes of the branch that the switch node needs to execute
for (Long switchTaskCode : switchTaskList) {
getSwitchNeedWorkCodes(switchTaskCode, dag, switchNeedWorkCodes);
}
// conditionResultVoList.remove(resultConditionLocation);
for (SwitchResultVo info : conditionResultVoList) {
if (CollectionUtils.isEmpty(info.getNextNode())) {
continue;
}
for (Long nextNode : info.getNextNode()) {
setSwitchTaskNodeSkip(nextNode, dag, completeTaskList, skipTaskNodeList,
switchNeedWorkCodes);
}
} }
return switchTaskList; if (nextBranch == null) {
log.error("switchParameters.getNextBranch() is null, please check the switch task configuration");
return Collections.emptyList();
} }
/** Set<Long> allNextBranches = new HashSet<>();
* get all downstream nodes of the branch that the switch node needs to execute if (switchResult.getNextNode() != null) {
* allNextBranches.add(switchResult.getNextNode());
* @param taskCode
* @param dag
* @param switchNeedWorkCodes
*/
public static void getSwitchNeedWorkCodes(Long taskCode, DAG<Long, TaskNode, TaskNodeRelation> dag,
Set<Long> switchNeedWorkCodes) {
switchNeedWorkCodes.add(taskCode);
Set<Long> subsequentNodes = dag.getSubsequentNodes(taskCode);
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(subsequentNodes)) {
for (Long subCode : subsequentNodes) {
getSwitchNeedWorkCodes(subCode, dag, switchNeedWorkCodes);
} }
if (CollectionUtils.isNotEmpty(switchResult.getDependTaskList())) {
for (SwitchResultVo switchResultVo : switchResult.getDependTaskList()) {
allNextBranches.add(switchResultVo.getNextNode());
} }
} }
private static void setSwitchTaskNodeSkip(Long skipNodeCode, allNextBranches.remove(nextBranch);
DAG<Long, TaskNode, TaskNodeRelation> dag,
Map<Long, TaskInstance> completeTaskList, for (Long branch : allNextBranches) {
Map<Long, TaskNode> skipTaskNodeList, setTaskNodeSkip(branch, dag, skipTaskNodeList);
Set<Long> switchNeedWorkCodes) {
// ignore when the node that needs to be skipped exists on the branch that the switch type node needs to execute
if (!dag.containsNode(skipNodeCode) || switchNeedWorkCodes.contains(skipNodeCode)) {
return;
}
skipTaskNodeList.putIfAbsent(skipNodeCode, dag.getNode(skipNodeCode));
Collection<Long> postNodeList = dag.getSubsequentNodes(skipNodeCode);
for (Long post : postNodeList) {
TaskNode postNode = dag.getNode(post);
if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) {
setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList);
}
} }
return Lists.newArrayList(nextBranch);
} }
/** /**
@ -485,7 +445,6 @@ public class DagHelper {
*/ */
private static void setTaskNodeSkip(Long skipNodeCode, private static void setTaskNodeSkip(Long skipNodeCode,
DAG<Long, TaskNode, TaskNodeRelation> dag, DAG<Long, TaskNode, TaskNodeRelation> dag,
Map<Long, TaskInstance> completeTaskList,
Map<Long, TaskNode> skipTaskNodeList) { Map<Long, TaskNode> skipTaskNodeList) {
if (!dag.containsNode(skipNodeCode)) { if (!dag.containsNode(skipNodeCode)) {
return; return;
@ -495,7 +454,7 @@ public class DagHelper {
for (Long post : postNodeList) { for (Long post : postNodeList) {
TaskNode postNode = dag.getNode(post); TaskNode postNode = dag.getNode(post);
if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) { if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) {
setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList); setTaskNodeSkip(post, dag, skipTaskNodeList);
} }
} }
} }
@ -591,31 +550,7 @@ public class DagHelper {
*/ */
public static boolean haveConditionsAfterNode(Long parentNodeCode, public static boolean haveConditionsAfterNode(Long parentNodeCode,
DAG<Long, TaskNode, TaskNodeRelation> dag) { DAG<Long, TaskNode, TaskNodeRelation> dag) {
return haveSubAfterNode(parentNodeCode, dag, TaskConstants.TASK_TYPE_CONDITIONS); return haveSubAfterNode(parentNodeCode, dag, ConditionsLogicTaskChannelFactory.NAME);
}
/**
* is there have conditions after the parent node
*/
public static boolean haveConditionsAfterNode(Long parentNodeCode, List<TaskNode> taskNodes) {
if (CollectionUtils.isEmpty(taskNodes)) {
return false;
}
for (TaskNode taskNode : taskNodes) {
List<Long> preTasksList = JSONUtils.toList(taskNode.getPreTasks(), Long.class);
if (preTasksList.contains(parentNodeCode) && taskNode.isConditionsTask()) {
return true;
}
}
return false;
}
/**
* is there have blocking node after the parent node
*/
public static boolean haveBlockingAfterNode(Long parentNodeCode,
DAG<Long, TaskNode, TaskNodeRelation> dag) {
return haveSubAfterNode(parentNodeCode, dag, TaskConstants.TASK_TYPE_BLOCKING);
} }
/** /**

457
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java

@ -17,19 +17,19 @@
package org.apache.dolphinscheduler.service.utils; package org.apache.dolphinscheduler.service.utils;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessDag; import org.apache.dolphinscheduler.service.process.ProcessDag;
@ -44,10 +44,9 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Lists;
import com.google.common.truth.Truth;
/**
* dag helper test
*/
public class DagHelperTest { public class DagHelperTest {
@Test @Test
@ -90,26 +89,105 @@ public class DagHelperTest {
boolean canSubmit = DagHelper.haveAllNodeAfterNode(parentNodeCode, dag); boolean canSubmit = DagHelper.haveAllNodeAfterNode(parentNodeCode, dag);
Assertions.assertTrue(canSubmit); Assertions.assertTrue(canSubmit);
boolean haveBlocking = DagHelper.haveBlockingAfterNode(parentNodeCode, dag);
Assertions.assertTrue(haveBlocking);
boolean haveConditions = DagHelper.haveConditionsAfterNode(parentNodeCode, dag); boolean haveConditions = DagHelper.haveConditionsAfterNode(parentNodeCode, dag);
Assertions.assertTrue(haveConditions); Assertions.assertTrue(haveConditions);
boolean dependent = DagHelper.haveSubAfterNode(parentNodeCode, dag, TaskConstants.TASK_TYPE_DEPENDENT); boolean dependent = DagHelper.haveSubAfterNode(parentNodeCode, dag, DependentLogicTaskChannelFactory.NAME);
Assertions.assertFalse(dependent); Assertions.assertFalse(dependent);
} }
/**
* test task node can submit
*
* @throws JsonProcessingException if error throws JsonProcessingException
*/
@Test @Test
public void testTaskNodeCanSubmit() throws IOException { public void testTaskNodeCanSubmit() {
List<TaskNode> taskNodeList = new ArrayList<>();
TaskNode node1 = new TaskNode();
node1.setId("1");
node1.setName("1");
node1.setCode(1);
node1.setType("SHELL");
taskNodeList.add(node1);
TaskNode node2 = new TaskNode();
node2.setId("2");
node2.setName("2");
node2.setCode(2);
node2.setType("SHELL");
List<String> dep2 = new ArrayList<>();
dep2.add("1");
node2.setPreTasks(JSONUtils.toJsonString(dep2));
taskNodeList.add(node2);
TaskNode node4 = new TaskNode();
node4.setId("4");
node4.setName("4");
node4.setCode(4);
node4.setType("SHELL");
taskNodeList.add(node4);
TaskNode node3 = new TaskNode();
node3.setId("3");
node3.setName("3");
node3.setCode(3);
node3.setType("SHELL");
List<String> dep3 = new ArrayList<>();
dep3.add("2");
dep3.add("4");
node3.setPreTasks(JSONUtils.toJsonString(dep3));
taskNodeList.add(node3);
TaskNode node5 = new TaskNode();
node5.setId("5");
node5.setName("5");
node5.setCode(5);
node5.setType("SHELL");
List<String> dep5 = new ArrayList<>();
dep5.add("3");
dep5.add("8");
node5.setPreTasks(JSONUtils.toJsonString(dep5));
taskNodeList.add(node5);
TaskNode node6 = new TaskNode();
node6.setId("6");
node6.setName("6");
node6.setCode(6);
node6.setType("SHELL");
List<String> dep6 = new ArrayList<>();
dep6.add("3");
node6.setPreTasks(JSONUtils.toJsonString(dep6));
taskNodeList.add(node6);
TaskNode node7 = new TaskNode();
node7.setId("7");
node7.setName("7");
node7.setCode(7);
node7.setType("SHELL");
List<String> dep7 = new ArrayList<>();
dep7.add("5");
node7.setPreTasks(JSONUtils.toJsonString(dep7));
taskNodeList.add(node7);
TaskNode node8 = new TaskNode();
node8.setId("8");
node8.setName("8");
node8.setCode(8);
node8.setType("SHELL");
List<String> dep8 = new ArrayList<>();
dep8.add("2");
node8.setPreTasks(JSONUtils.toJsonString(dep8));
taskNodeList.add(node8);
List<Long> startNodes = new ArrayList<>();
List<Long> recoveryNodes = new ArrayList<>();
List<TaskNode> destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList,
startNodes, recoveryNodes, TaskDependType.TASK_POST);
List<TaskNodeRelation> taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList);
ProcessDag processDag = new ProcessDag();
processDag.setEdges(taskNodeRelations);
processDag.setNodes(destTaskNodeList);
// 1->2->3->5->7 // 1->2->3->5->7
// 4->3->6 // 4->3->6
DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag(); // 1->2->8->5->7
DAG<Long, TaskNode, TaskNodeRelation> dag = DagHelper.buildDagGraph(processDag);
TaskNode taskNode3 = dag.getNode(3L); TaskNode taskNode3 = dag.getNode(3L);
Map<Long, TaskInstance> completeTaskList = new HashMap<>(); Map<Long, TaskInstance> completeTaskList = new HashMap<>();
Map<Long, TaskNode> skipNodeList = new HashMap<>(); Map<Long, TaskNode> skipNodeList = new HashMap<>();
@ -117,7 +195,7 @@ public class DagHelperTest {
Boolean canSubmit = false; Boolean canSubmit = false;
// 2/4 are forbidden submit 3 // 2/4 are forbidden submit 3
TaskNode node2 = dag.getNode(2L); node2 = dag.getNode(2L);
node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
TaskNode nodex = dag.getNode(4L); TaskNode nodex = dag.getNode(4L);
nodex.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); nodex.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
@ -132,21 +210,107 @@ public class DagHelperTest {
Assertions.assertEquals(canSubmit, false); Assertions.assertEquals(canSubmit, false);
// 2/3 forbidden submit 5 // 2/3 forbidden submit 5
TaskNode node3 = dag.getNode(3L); node3 = dag.getNode(3L);
node3.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); node3.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
TaskNode node8 = dag.getNode(8L); node8 = dag.getNode(8L);
node8.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); node8.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
TaskNode node5 = dag.getNode(5L); node5 = dag.getNode(5L);
canSubmit = DagHelper.allDependsForbiddenOrEnd(node5, dag, skipNodeList, completeTaskList); canSubmit = DagHelper.allDependsForbiddenOrEnd(node5, dag, skipNodeList, completeTaskList);
Assertions.assertEquals(canSubmit, true); Assertions.assertEquals(canSubmit, true);
} }
/**
* test parse post node list
*/
@Test @Test
public void testParsePostNodeList() throws IOException { public void testParsePostNodeList() {
DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag(); List<TaskNode> taskNodeList = new ArrayList<>();
TaskNode node1 = new TaskNode();
node1.setId("1");
node1.setName("1");
node1.setCode(1);
node1.setType("SHELL");
taskNodeList.add(node1);
TaskNode node2 = new TaskNode();
node2.setId("2");
node2.setName("2");
node2.setCode(2);
node2.setType("SHELL");
List<String> dep2 = new ArrayList<>();
dep2.add("1");
node2.setPreTasks(JSONUtils.toJsonString(dep2));
taskNodeList.add(node2);
TaskNode node4 = new TaskNode();
node4.setId("4");
node4.setName("4");
node4.setCode(4);
node4.setType("SHELL");
taskNodeList.add(node4);
TaskNode node3 = new TaskNode();
node3.setId("3");
node3.setName("3");
node3.setCode(3);
node3.setType("SHELL");
List<String> dep3 = new ArrayList<>();
dep3.add("2");
dep3.add("4");
node3.setPreTasks(JSONUtils.toJsonString(dep3));
taskNodeList.add(node3);
TaskNode node5 = new TaskNode();
node5.setId("5");
node5.setName("5");
node5.setCode(5);
node5.setType("SHELL");
List<String> dep5 = new ArrayList<>();
dep5.add("3");
dep5.add("8");
node5.setPreTasks(JSONUtils.toJsonString(dep5));
taskNodeList.add(node5);
TaskNode node6 = new TaskNode();
node6.setId("6");
node6.setName("6");
node6.setCode(6);
node6.setType("SHELL");
List<String> dep6 = new ArrayList<>();
dep6.add("3");
node6.setPreTasks(JSONUtils.toJsonString(dep6));
taskNodeList.add(node6);
TaskNode node7 = new TaskNode();
node7.setId("7");
node7.setName("7");
node7.setCode(7);
node7.setType("SHELL");
List<String> dep7 = new ArrayList<>();
dep7.add("5");
node7.setPreTasks(JSONUtils.toJsonString(dep7));
taskNodeList.add(node7);
TaskNode node8 = new TaskNode();
node8.setId("8");
node8.setName("8");
node8.setCode(8);
node8.setType("SHELL");
List<String> dep8 = new ArrayList<>();
dep8.add("2");
node8.setPreTasks(JSONUtils.toJsonString(dep8));
taskNodeList.add(node8);
List<Long> startNodes = new ArrayList<>();
List<Long> recoveryNodes = new ArrayList<>();
List<TaskNode> destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList,
startNodes, recoveryNodes, TaskDependType.TASK_POST);
List<TaskNodeRelation> taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList);
ProcessDag processDag = new ProcessDag();
processDag.setEdges(taskNodeRelations);
processDag.setNodes(destTaskNodeList);
// 1->2->3->5->7
// 4->3->6
// 1->2->8->5->7
DAG<Long, TaskNode, TaskNodeRelation> dag = DagHelper.buildDagGraph(processDag);
Map<Long, TaskInstance> completeTaskList = new HashMap<>(); Map<Long, TaskInstance> completeTaskList = new HashMap<>();
Map<Long, TaskNode> skipNodeList = new HashMap<>(); Map<Long, TaskNode> skipNodeList = new HashMap<>();
@ -206,11 +370,6 @@ public class DagHelperTest {
Assertions.assertTrue(postNodes.contains(7L)); Assertions.assertTrue(postNodes.contains(7L));
} }
/**
* test forbidden post node
*
* @throws JsonProcessingException
*/
@Test @Test
public void testForbiddenPostNode() throws IOException { public void testForbiddenPostNode() throws IOException {
DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag(); DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag();
@ -244,11 +403,6 @@ public class DagHelperTest {
Assertions.assertTrue(postNodes.contains(3L)); Assertions.assertTrue(postNodes.contains(3L));
} }
/**
* test condition post node
*
* @throws JsonProcessingException
*/
@Test @Test
public void testConditionPostNode() throws IOException { public void testConditionPostNode() throws IOException {
DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag(); DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag();
@ -262,26 +416,18 @@ public class DagHelperTest {
completeTaskList.put(4L, new TaskInstance()); completeTaskList.put(4L, new TaskInstance());
TaskInstance taskInstance3 = new TaskInstance(); TaskInstance taskInstance3 = new TaskInstance();
taskInstance3.setTaskType(TASK_TYPE_CONDITIONS); taskInstance3.setTaskType(ConditionsLogicTaskChannelFactory.NAME);
Map<String, Object> params = new HashMap<>(); ConditionsParameters.ConditionResult conditionResult = ConditionsParameters.ConditionResult.builder()
.conditionSuccess(true)
.successNode(Lists.newArrayList(5L))
.failedNode(Lists.newArrayList(6L))
.build();
ConditionsParameters conditionsParameters = new ConditionsParameters(); ConditionsParameters conditionsParameters = new ConditionsParameters();
conditionsParameters.setConditionSuccess(true); conditionsParameters.setConditionResult(conditionResult);
params.put(Constants.DEPENDENCE, "{\"conditionSuccess\": true}"); taskInstance3.setTaskParams(JSONUtils.toJsonString(conditionsParameters));
params.put(Constants.CONDITION_RESULT, "{\n"
+
" \"successNode\": [5\n"
+
" ],\n"
+
" \"failedNode\": [6\n"
+
" ]\n"
+
" }");
taskInstance3.setTaskParams(JSONUtils.toJsonString(params));
taskInstance3.setState(TaskExecutionStatus.SUCCESS); taskInstance3.setState(TaskExecutionStatus.SUCCESS);
TaskNode node3 = dag.getNode(3L); TaskNode node3 = dag.getNode(3L);
node3.setType(TASK_TYPE_CONDITIONS); node3.setType(ConditionsLogicTaskChannelFactory.NAME);
// complete 1/2/3/4 expect:8 // complete 1/2/3/4 expect:8
completeTaskList.put(3L, taskInstance3); completeTaskList.put(3L, taskInstance3);
postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList);
@ -308,23 +454,106 @@ public class DagHelperTest {
} }
@Test @Test
public void testSwitchPostNode() throws IOException { public void testSwitchPostNode() {
DAG<Long, TaskNode, TaskNodeRelation> dag = generateDag2(); List<TaskNode> taskNodeList = new ArrayList<>();
TaskNode node = new TaskNode();
node.setId("0");
node.setName("0");
node.setCode(0);
node.setType("SHELL");
taskNodeList.add(node);
TaskNode node1 = new TaskNode();
node1.setId("1");
node1.setName("1");
node1.setCode(1);
node1.setType(SwitchLogicTaskChannelFactory.NAME);
SwitchParameters switchParameters = new SwitchParameters();
node1.setParams(JSONUtils.toJsonString(switchParameters));
taskNodeList.add(node1);
TaskNode node2 = new TaskNode();
node2.setId("2");
node2.setName("2");
node2.setCode(2);
node2.setType("SHELL");
List<String> dep2 = new ArrayList<>();
dep2.add("1");
node2.setPreTasks(JSONUtils.toJsonString(dep2));
taskNodeList.add(node2);
TaskNode node4 = new TaskNode();
node4.setId("4");
node4.setName("4");
node4.setCode(4);
node4.setType("SHELL");
List<String> dep4 = new ArrayList<>();
dep4.add("1");
node4.setPreTasks(JSONUtils.toJsonString(dep4));
taskNodeList.add(node4);
TaskNode node5 = new TaskNode();
node5.setId("5");
node5.setName("5");
node5.setCode(5);
node5.setType("SHELL");
List<Long> dep5 = new ArrayList<>();
dep5.add(1L);
node5.setPreTasks(JSONUtils.toJsonString(dep5));
taskNodeList.add(node5);
TaskNode node6 = new TaskNode();
node5.setId("6");
node5.setName("6");
node5.setCode(6);
node5.setType("SHELL");
List<Long> dep6 = new ArrayList<>();
dep5.add(2L);
dep5.add(4L);
node5.setPreTasks(JSONUtils.toJsonString(dep6));
taskNodeList.add(node6);
List<Long> startNodes = new ArrayList<>();
List<Long> recoveryNodes = new ArrayList<>();
// 0
// 1->2->6
// 1->4->6
// 1->5
List<TaskNode> destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList,
startNodes, recoveryNodes, TaskDependType.TASK_POST);
List<TaskNodeRelation> taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList);
ProcessDag processDag = new ProcessDag();
processDag.setEdges(taskNodeRelations);
processDag.setNodes(destTaskNodeList);
DAG<Long, TaskNode, TaskNodeRelation> dag = DagHelper.buildDagGraph(processDag);
Map<Long, TaskNode> skipTaskNodeList = new HashMap<>(); Map<Long, TaskNode> skipTaskNodeList = new HashMap<>();
Map<Long, TaskInstance> completeTaskList = new HashMap<>(); Map<Long, TaskInstance> completeTaskList = new HashMap<>();
completeTaskList.put(0l, new TaskInstance()); completeTaskList.put(0L, new TaskInstance());
TaskInstance taskInstance = new TaskInstance(); TaskInstance taskInstance = new TaskInstance();
taskInstance.setState(TaskExecutionStatus.SUCCESS); taskInstance.setState(TaskExecutionStatus.SUCCESS);
taskInstance.setTaskCode(1l); taskInstance.setTaskCode(1L);
Map<String, Object> taskParamsMap = new HashMap<>(); taskInstance.setTaskType(SwitchLogicTaskChannelFactory.NAME);
taskParamsMap.put(Constants.SWITCH_RESULT, ""); switchParameters = SwitchParameters.builder()
taskInstance.setTaskParams(JSONUtils.toJsonString(taskParamsMap)); .nextBranch(5L)
taskInstance.setSwitchDependency(getSwitchNode()); .switchResult(SwitchParameters.SwitchResult.builder()
.dependTaskList(Lists.newArrayList(
new SwitchResultVo("", 2L),
new SwitchResultVo("", 4L)))
.nextNode(5L)
.build())
.build();
taskInstance.setTaskParams(JSONUtils.toJsonString(switchParameters));
completeTaskList.put(1l, taskInstance); completeTaskList.put(1l, taskInstance);
DagHelper.skipTaskNode4Switch(dag.getNode(1l), skipTaskNodeList, completeTaskList, dag); List<Long> nextBranch = DagHelper.skipTaskNode4Switch(skipTaskNodeList, taskInstance, dag);
Assertions.assertNotNull(skipTaskNodeList.get(2L)); Assertions.assertNotNull(skipTaskNodeList.get(2L));
Assertions.assertEquals(1, skipTaskNodeList.size()); Assertions.assertNotNull(skipTaskNodeList.get(4L));
Assertions.assertEquals(2, skipTaskNodeList.size());
Truth.assertThat(nextBranch).containsExactly(5L);
} }
/** /**
* process: * process:
* 1->2->3->5->7 * 1->2->3->5->7
@ -429,108 +658,6 @@ public class DagHelperTest {
return DagHelper.buildDagGraph(processDag); return DagHelper.buildDagGraph(processDag);
} }
/**
* DAG graph:
* -> 2->
* / \
* / \
* 0->1(switch)->5 6
* \ /
* \ /
* -> 4->
*
* @return dag
* @throws JsonProcessingException if error throws JsonProcessingException
*/
private DAG<Long, TaskNode, TaskNodeRelation> generateDag2() throws IOException {
List<TaskNode> taskNodeList = new ArrayList<>();
TaskNode node = new TaskNode();
node.setId("0");
node.setName("0");
node.setCode(0);
node.setType("SHELL");
taskNodeList.add(node);
TaskNode node1 = new TaskNode();
node1.setId("1");
node1.setName("1");
node1.setCode(1);
node1.setType("switch");
node1.setDependence(JSONUtils.toJsonString(getSwitchNode()));
taskNodeList.add(node1);
TaskNode node2 = new TaskNode();
node2.setId("2");
node2.setName("2");
node2.setCode(2);
node2.setType("SHELL");
List<String> dep2 = new ArrayList<>();
dep2.add("1");
node2.setPreTasks(JSONUtils.toJsonString(dep2));
taskNodeList.add(node2);
TaskNode node4 = new TaskNode();
node4.setId("4");
node4.setName("4");
node4.setCode(4);
node4.setType("SHELL");
List<String> dep4 = new ArrayList<>();
dep4.add("1");
node4.setPreTasks(JSONUtils.toJsonString(dep4));
taskNodeList.add(node4);
TaskNode node5 = new TaskNode();
node5.setId("5");
node5.setName("5");
node5.setCode(5);
node5.setType("SHELL");
List<Long> dep5 = new ArrayList<>();
dep5.add(1L);
node5.setPreTasks(JSONUtils.toJsonString(dep5));
taskNodeList.add(node5);
TaskNode node6 = new TaskNode();
node5.setId("6");
node5.setName("6");
node5.setCode(6);
node5.setType("SHELL");
List<Long> dep6 = new ArrayList<>();
dep5.add(2L);
dep5.add(4L);
node5.setPreTasks(JSONUtils.toJsonString(dep6));
taskNodeList.add(node6);
List<Long> startNodes = new ArrayList<>();
List<Long> recoveryNodes = new ArrayList<>();
List<TaskNode> destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList,
startNodes, recoveryNodes, TaskDependType.TASK_POST);
List<TaskNodeRelation> taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList);
ProcessDag processDag = new ProcessDag();
processDag.setEdges(taskNodeRelations);
processDag.setNodes(destTaskNodeList);
return DagHelper.buildDagGraph(processDag);
}
private SwitchParameters getSwitchNode() {
SwitchParameters conditionsParameters = new SwitchParameters();
SwitchResultVo switchResultVo1 = new SwitchResultVo();
switchResultVo1.setCondition(" 2 == 1");
switchResultVo1.setNextNode(2L);
SwitchResultVo switchResultVo2 = new SwitchResultVo();
switchResultVo2.setCondition(" 2 == 2");
switchResultVo2.setNextNode(4L);
List<SwitchResultVo> list = new ArrayList<>();
list.add(switchResultVo1);
list.add(switchResultVo2);
conditionsParameters.setDependTaskList(list);
conditionsParameters.setNextNode(5L);
conditionsParameters.setRelation("AND");
conditionsParameters.setResultConditionLocation(1);
// in: AND(AND(1 is SUCCESS))
return conditionsParameters;
}
@Test @Test
public void testBuildDagGraph() { public void testBuildDagGraph() {
String shellJson = String shellJson =

7
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java

@ -78,8 +78,11 @@ public class HdfsStorageOperator extends AbstractStorageOperator implements Clos
private void initHdfsPath() { private void initHdfsPath() {
Path path = new Path(resourceBaseAbsolutePath); Path path = new Path(resourceBaseAbsolutePath);
if (!fs.exists(path)) { if (!fs.exists(path)) {
fs.mkdirs(path); if (!fs.mkdirs(path)) {
log.info("Create hdfs path: {}", path); log.info("Create hdfs path: {} failed", path);
} else {
log.error("Create hdfs path: {} success", path);
}
} }
} }

5
dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml

@ -297,5 +297,10 @@
<groupId>org.projectlombok</groupId> <groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId> <artifactId>lombok</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
</project> </project>

21
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java → dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ILogicTaskChannel.java

@ -15,21 +15,10 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.enums; package org.apache.dolphinscheduler.plugin.task.api;
public enum BlockingOpportunity {
BLOCKING_ON_SUCCESS("BlockingOnSuccess"),
BLOCKING_ON_FAILED("BlockingOnFailed");
private final String desc;
BlockingOpportunity(String desc) {
this.desc = desc;
}
public String getDesc() {
return desc;
}
/**
* Used to mark a task channel as a logic task channel, the logic task channel is a special task channel that will be executed at master.
*/
public interface ILogicTaskChannel extends TaskChannel {
} }

10
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannel.java

@ -18,19 +18,11 @@
package org.apache.dolphinscheduler.plugin.task.api; package org.apache.dolphinscheduler.plugin.task.api;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public interface TaskChannel { public interface TaskChannel {
void cancelApplication(boolean status);
// todo: return ITask
AbstractTask createTask(TaskExecutionContext taskRequest); AbstractTask createTask(TaskExecutionContext taskRequest);
// todo: return IParameters AbstractParameters parseParameters(String taskParams);
AbstractParameters parseParameters(ParametersNode parametersNode);
ResourceParametersHelper getResources(String parameters);
} }

13
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java

@ -17,16 +17,19 @@
package org.apache.dolphinscheduler.plugin.task.api; package org.apache.dolphinscheduler.plugin.task.api;
import org.apache.dolphinscheduler.spi.common.UiChannelFactory;
import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; import org.apache.dolphinscheduler.spi.plugin.PrioritySPI;
import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import org.apache.dolphinscheduler.spi.plugin.SPIIdentify;
public interface TaskChannelFactory extends UiChannelFactory, PrioritySPI { public interface TaskChannelFactory extends PrioritySPI {
TaskChannel create();
default SPIIdentify getIdentify() { default SPIIdentify getIdentify() {
return SPIIdentify.builder().name(getName()).build(); return SPIIdentify.builder()
.name(getName())
.build();
} }
String getName();
TaskChannel create();
} }

21
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java

@ -362,24 +362,10 @@ public class TaskConstants {
*/ */
public static final String DATA_QUALITY_JAR_DIR = "data-quality.jar.dir"; public static final String DATA_QUALITY_JAR_DIR = "data-quality.jar.dir";
public static final String TASK_TYPE_CONDITIONS = "CONDITIONS";
public static final String TASK_TYPE_SWITCH = "SWITCH";
public static final String TASK_TYPE_SUB_PROCESS = "SUB_PROCESS";
public static final String TASK_TYPE_DYNAMIC = "DYNAMIC";
public static final String TASK_TYPE_DEPENDENT = "DEPENDENT";
public static final String TASK_TYPE_SQL = "SQL";
public static final String TASK_TYPE_DATA_QUALITY = "DATA_QUALITY"; public static final String TASK_TYPE_DATA_QUALITY = "DATA_QUALITY";
public static final Set<String> TASK_TYPE_SET_K8S = Sets.newHashSet("K8S", "KUBEFLOW"); public static final Set<String> TASK_TYPE_SET_K8S = Sets.newHashSet("K8S", "KUBEFLOW");
public static final String TASK_TYPE_BLOCKING = "BLOCKING";
/** /**
* azure config * azure config
*/ */
@ -389,13 +375,6 @@ public class TaskConstants {
public static final String AZURE_SECRET_TENANT_ID = "resource.azure.tenant.id"; public static final String AZURE_SECRET_TENANT_ID = "resource.azure.tenant.id";
public static final String QUERY_INTERVAL = "resource.query.interval"; public static final String QUERY_INTERVAL = "resource.query.interval";
/**
* aws config
*/
public static final String AWS_ACCESS_KEY_ID = "resource.aws.access.key.id";
public static final String AWS_SECRET_ACCESS_KEY = "resource.aws.secret.access.key";
public static final String AWS_REGION = "resource.aws.region";
/** /**
* alibaba cloud config * alibaba cloud config
*/ */

96
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java

@ -17,21 +17,13 @@
package org.apache.dolphinscheduler.plugin.task.api; package org.apache.dolphinscheduler.plugin.task.api;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -39,15 +31,15 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class TaskPluginManager { public class TaskPluginManager {
private static final Map<String, TaskChannelFactory> taskChannelFactoryMap = new HashMap<>();
private static final Map<String, TaskChannel> taskChannelMap = new HashMap<>(); private static final Map<String, TaskChannel> taskChannelMap = new HashMap<>();
private static final AtomicBoolean loadedFlag = new AtomicBoolean(false); private static final AtomicBoolean loadedFlag = new AtomicBoolean(false);
/** static {
* Load task plugins from classpath. loadTaskPlugin();
*/ }
public static void loadPlugin() {
public static void loadTaskPlugin() {
if (!loadedFlag.compareAndSet(false, true)) { if (!loadedFlag.compareAndSet(false, true)) {
log.warn("The task plugin has already been loaded"); log.warn("The task plugin has already been loaded");
return; return;
@ -59,7 +51,6 @@ public class TaskPluginManager {
log.info("Registering task plugin: {} - {}", factoryName, factory.getClass().getSimpleName()); log.info("Registering task plugin: {} - {}", factoryName, factory.getClass().getSimpleName());
taskChannelFactoryMap.put(factoryName, factory);
taskChannelMap.put(factoryName, factory.create()); taskChannelMap.put(factoryName, factory.create());
log.info("Registered task plugin: {} - {}", factoryName, factory.getClass().getSimpleName()); log.info("Registered task plugin: {} - {}", factoryName, factory.getClass().getSimpleName());
@ -67,48 +58,49 @@ public class TaskPluginManager {
} }
public static Map<String, TaskChannel> getTaskChannelMap() { /**
return Collections.unmodifiableMap(taskChannelMap); * Get the TaskChannel by type, if the TaskChannel is not found, will throw
} * @param type task type, cannot be null
* @throws IllegalArgumentException if the TaskChannel is not found
public static Map<String, TaskChannelFactory> getTaskChannelFactoryMap() { */
return Collections.unmodifiableMap(taskChannelFactoryMap);
}
public static TaskChannel getTaskChannel(String type) { public static TaskChannel getTaskChannel(String type) {
return getTaskChannelMap().get(type); checkNotNull(type, "type cannot be null");
TaskChannel taskChannel = taskChannelMap.get(type);
if (taskChannel == null) {
throw new IllegalArgumentException("Cannot find TaskChannel for : " + type);
} }
return taskChannel;
public static boolean checkTaskParameters(ParametersNode parametersNode) {
AbstractParameters abstractParameters = getParameters(parametersNode);
return abstractParameters != null && abstractParameters.checkParameters();
} }
public static AbstractParameters getParameters(ParametersNode parametersNode) { /**
String taskType = parametersNode.getTaskType(); * Check if the task parameters is validated
if (Objects.isNull(taskType)) { * @param taskType task type, cannot be null
return null; * @param taskParams task parameters
} * @return true if the task parameters is validated, otherwise false
switch (taskType) { * @throws IllegalArgumentException if the TaskChannel is not found
case TaskConstants.TASK_TYPE_CONDITIONS: * @throws IllegalArgumentException if cannot deserialize the task parameters
return JSONUtils.parseObject(parametersNode.getTaskParams(), ConditionsParameters.class); */
case TaskConstants.TASK_TYPE_SWITCH: public static boolean checkTaskParameters(String taskType, String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), SwitchParameters.class); AbstractParameters abstractParameters = parseTaskParameters(taskType, taskParams);
case TaskConstants.TASK_TYPE_SUB_PROCESS: return abstractParameters.checkParameters();
return JSONUtils.parseObject(parametersNode.getTaskParams(), SubProcessParameters.class);
case TaskConstants.TASK_TYPE_DEPENDENT:
return JSONUtils.parseObject(parametersNode.getTaskParams(), DependentParameters.class);
case TaskConstants.TASK_TYPE_BLOCKING:
return JSONUtils.parseObject(parametersNode.getTaskParams(), BlockingParameters.class);
case TaskConstants.TASK_TYPE_DYNAMIC:
return JSONUtils.parseObject(parametersNode.getTaskParams(), DynamicParameters.class);
default:
TaskChannel taskChannel = getTaskChannelMap().get(taskType);
if (Objects.isNull(taskChannel)) {
return null;
} }
return taskChannel.parseParameters(parametersNode);
/**
* Parse the task parameters
* @param taskType task type, cannot be null
* @param taskParams task parameters
* @return AbstractParameters
* @throws IllegalArgumentException if the TaskChannel is not found
* @throws IllegalArgumentException if cannot deserialize the task parameters
*/
public static AbstractParameters parseTaskParameters(String taskType, String taskParams) {
checkNotNull(taskType, "taskType cannot be null");
TaskChannel taskChannel = getTaskChannel(taskType);
AbstractParameters abstractParameters = taskChannel.parseParameters(taskParams);
if (abstractParameters == null) {
throw new IllegalArgumentException("Cannot parse task parameters: " + taskParams + " for : " + taskType);
} }
return abstractParameters;
} }
} }

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DynamicInputParameter.java

@ -17,12 +17,16 @@
package org.apache.dolphinscheduler.plugin.task.api.model; package org.apache.dolphinscheduler.plugin.task.api.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.NonNull; import lombok.NonNull;
@Data @Data
@Builder
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor
public class DynamicInputParameter { public class DynamicInputParameter {
@NonNull @NonNull

15
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java

@ -17,9 +17,6 @@
package org.apache.dolphinscheduler.plugin.task.api.model; package org.apache.dolphinscheduler.plugin.task.api.model;
import java.util.ArrayList;
import java.util.List;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@ -30,15 +27,5 @@ import lombok.NoArgsConstructor;
public class SwitchResultVo { public class SwitchResultVo {
private String condition; private String condition;
private List<Long> nextNode; private Long nextNode;
public void setNextNode(Object nextNode) {
if (nextNode instanceof Long) {
List<Long> nextNodeList = new ArrayList<>();
nextNodeList.add((Long) nextNode);
this.nextNode = nextNodeList;
} else {
this.nextNode = (ArrayList) nextNode;
}
}
} }

51
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.commons.lang3.StringUtils;
public class BlockingParameters extends AbstractParameters {
// condition of blocking: BlockingOnFailed or BlockingOnSuccess
private String blockingOpportunity;
// if true, alert when blocking, otherwise do nothing
private boolean isAlertWhenBlocking;
@Override
public boolean checkParameters() {
return !StringUtils.isEmpty(blockingOpportunity);
}
public String getBlockingOpportunity() {
return blockingOpportunity;
}
public void setBlockingCondition(String blockingOpportunity) {
this.blockingOpportunity = blockingOpportunity;
}
public boolean isAlertWhenBlocking() {
return isAlertWhenBlocking;
}
public void setAlertWhenBlocking(boolean alertWhenBlocking) {
isAlertWhenBlocking = alertWhenBlocking;
}
}

41
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java

@ -19,36 +19,59 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import java.util.ArrayList; import org.apache.commons.collections4.CollectionUtils;
import java.util.List; import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
@Data @Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ConditionsParameters extends AbstractParameters { public class ConditionsParameters extends AbstractParameters {
// depend node list and state, only need task name private ConditionDependency dependence;
private List<DependentTaskModel> dependTaskList;
private DependentRelation relation;
private boolean conditionSuccess; private ConditionResult conditionResult;
@Override @Override
public boolean checkParameters() { public boolean checkParameters() {
if (dependence == null || CollectionUtils.isEmpty(dependence.getDependTaskList())) {
return false;
}
if (conditionResult == null || CollectionUtils.isEmpty(conditionResult.getSuccessNode())
|| CollectionUtils.isEmpty(conditionResult.getFailedNode())) {
return false;
}
return true; return true;
} }
@Override @Data
public List<ResourceInfo> getResourceFilesList() { @Builder
return new ArrayList<>(); @NoArgsConstructor
@AllArgsConstructor
public static class ConditionDependency {
private List<DependentTaskModel> dependTaskList;
private DependentRelation relation;
} }
@Data @Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public static class ConditionResult { public static class ConditionResult {
private boolean conditionSuccess;
private List<Long> successNode; private List<Long> successNode;
private List<Long> failedNode; private List<Long> failedNode;
} }

39
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java

@ -20,26 +20,53 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.commons.collections4.CollectionUtils;
import java.util.List; import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
public class DependentParameters extends AbstractParameters { public class DependentParameters extends AbstractParameters {
private Dependence dependence;
@Override
public boolean checkParameters() {
if (dependence == null) {
return false;
}
if (CollectionUtils.isEmpty(dependence.getDependTaskList())) {
return false;
}
if (dependence.getCheckInterval() != null && dependence.getCheckInterval() <= 0) {
return false;
}
return true;
}
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public static class Dependence {
private List<DependentTaskModel> dependTaskList; private List<DependentTaskModel> dependTaskList;
private DependentRelation relation; private DependentRelation relation;
/** Time unit is second */ /**
* Time unit is second
*/
private Integer checkInterval; private Integer checkInterval;
private DependentFailurePolicyEnum failurePolicy; private DependentFailurePolicyEnum failurePolicy;
/** Time unit is minutes */ /**
* Time unit is minutes
*/
private Integer failureWaitingTime; private Integer failureWaitingTime;
@Override
public boolean checkParameters() {
return true;
} }
/** /**

6
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DynamicParameters.java

@ -21,9 +21,15 @@ import org.apache.dolphinscheduler.plugin.task.api.model.DynamicInputParameter;
import java.util.List; import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
@Data @Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DynamicParameters extends AbstractParameters { public class DynamicParameters extends AbstractParameters {
/** /**

116
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java

@ -1,116 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.parameters;
/**
* TODO <p>Need to optimize, why there are multiple task parameter variables:taskParams,dependence,switchResult</p>
*
*/
public class ParametersNode {
private String taskType;
private String taskParams;
private String dependence;
private String switchResult;
public static ParametersNode.ParametersNodeBuilder builder() {
return new ParametersNode.ParametersNodeBuilder();
}
public static class ParametersNodeBuilder {
private String taskType;
private String taskParams;
private String dependence;
private String switchResult;
public ParametersNodeBuilder taskType(String taskType) {
this.taskType = taskType;
return this;
}
public ParametersNodeBuilder taskParams(String taskParams) {
this.taskParams = taskParams;
return this;
}
public ParametersNodeBuilder dependence(String dependence) {
this.dependence = dependence;
return this;
}
public ParametersNodeBuilder switchResult(String switchResult) {
this.switchResult = switchResult;
return this;
}
public ParametersNode build() {
return new ParametersNode(this.taskType, this.taskParams, this.dependence, this.switchResult);
}
}
public ParametersNode() {
}
public ParametersNode(String taskType, String taskParams, String dependence, String switchResult) {
this.taskType = taskType;
this.taskParams = taskParams;
this.dependence = dependence;
this.switchResult = switchResult;
}
public String getTaskType() {
return taskType;
}
public void setTaskType(String taskType) {
this.taskType = taskType;
}
public String getTaskParams() {
return taskParams;
}
public void setTaskParams(String taskParams) {
this.taskParams = taskParams;
}
public String getDependence() {
return dependence;
}
public void setDependence(String dependence) {
this.dependence = dependence;
}
public String getSwitchResult() {
return switchResult;
}
public void setSwitchResult(String switchResult) {
this.switchResult = switchResult;
}
}

5
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters;
public class SubProcessParameters extends AbstractParameters { public class SubProcessParameters extends AbstractParameters {
/**
* process definition id
*/
private long processDefinitionCode; private long processDefinitionCode;
public void setProcessDefinitionCode(long processDefinitionCode) { public void setProcessDefinitionCode(long processDefinitionCode) {
@ -34,7 +31,7 @@ public class SubProcessParameters extends AbstractParameters {
@Override @Override
public boolean checkParameters() { public boolean checkParameters() {
return this.processDefinitionCode != 0; return this.processDefinitionCode > 0;
} }
} }

79
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java

@ -17,69 +17,56 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters; package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation;
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
import java.util.ArrayList; import org.apache.commons.collections4.CollectionUtils;
import java.util.List; import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class SwitchParameters extends AbstractParameters { public class SwitchParameters extends AbstractParameters {
private DependentRelation dependRelation; // due to history reasons, the field name is switchResult
private String relation; private SwitchResult switchResult;
private List<Long> nextNode;
// The next branch which should be executed after the switch logic task executed.
private Long nextBranch;
@Override @Override
public boolean checkParameters() { public boolean checkParameters() {
return true; if (switchResult == null) {
} return false;
private int resultConditionLocation;
private List<SwitchResultVo> dependTaskList;
public DependentRelation getDependRelation() {
return dependRelation;
}
public void setDependRelation(DependentRelation dependRelation) {
this.dependRelation = dependRelation;
} }
if (CollectionUtils.isEmpty(switchResult.getDependTaskList()) && switchResult.getNextNode() == null) {
public int getResultConditionLocation() { return false;
return resultConditionLocation;
} }
for (SwitchResultVo switchResultVo : switchResult.getDependTaskList()) {
public void setResultConditionLocation(int resultConditionLocation) { if (switchResultVo == null || switchResultVo.getNextNode() == null) {
this.resultConditionLocation = resultConditionLocation; return false;
} }
public String getRelation() {
return relation;
} }
return true;
public void setRelation(String relation) {
this.relation = relation;
} }
public List<SwitchResultVo> getDependTaskList() { @Data
return dependTaskList; @Builder
} @AllArgsConstructor
@NoArgsConstructor
public static class SwitchResult {
public void setDependTaskList(List<SwitchResultVo> dependTaskList) { // switch condition
this.dependTaskList = dependTaskList; private List<SwitchResultVo> dependTaskList;
}
public List<Long> getNextNode() { // default branch node code in switch task
return nextNode; private Long nextNode;
} }
public void setNextNode(Object nextNode) {
if (nextNode instanceof Long) {
List<Long> nextNodeList = new ArrayList<>();
nextNodeList.add((Long) nextNode);
this.nextNode = nextNodeList;
} else {
this.nextNode = (ArrayList) nextNode;
}
}
} }

5
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java

@ -17,13 +17,8 @@
package org.apache.dolphinscheduler.plugin.task.api.stream; package org.apache.dolphinscheduler.plugin.task.api.stream;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
public interface StreamTaskChannel extends TaskChannel { public interface StreamTaskChannel extends TaskChannel {
AbstractTask pauseTask(TaskExecutionContext taskExecutionContext);
AbstractTask recoverTask(TaskExecutionContext taskExecutionContext);
} }

48
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java → dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/AbstractLogicTaskChannel.java

@ -15,47 +15,17 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.entity; package org.apache.dolphinscheduler.plugin.task.api.task;
import java.util.Date; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.ILogicTaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import lombok.Data; public abstract class AbstractLogicTaskChannel implements ILogicTaskChannel {
@Data @Override
public class WorkerServer { public AbstractTask createTask(TaskExecutionContext taskRequest) {
throw new UnsupportedOperationException("createTask is not supported");
}
/**
* id
*/
private int id;
/**
* host
*/
private String host;
/**
* port
*/
private int port;
/**
* zookeeper directory
*/
private String zkDirectory;
/**
* resource info
*/
private String resInfo;
/**
* create time
*/
private Date createTime;
/**
* last heart beat time
*/
private Date lastHeartbeatTime;
} }

31
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannel.java

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
public class ConditionsLogicTaskChannel extends AbstractLogicTaskChannel {
@Override
public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(taskParams, ConditionsParameters.class);
}
}

34
dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java → dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannelFactory.java

@ -15,29 +15,25 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.spi.common; package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import java.util.List; import com.google.auto.service.AutoService;
public interface UiChannelFactory { @AutoService(TaskChannelFactory.class)
public class ConditionsLogicTaskChannelFactory implements TaskChannelFactory {
/** public static final String NAME = "CONDITIONS";
* plugin name
* Must be UNIQUE .
* This alert plugin name eg: email , message ...
* Name can often be displayed on the page ui eg : email , message , MR , spark , hive ...
*
* @return this alert plugin name
*/
String getName();
/** @Override
* Returns the configurable parameters that this plugin needs to display on the web ui public String getName() {
* return NAME;
* @return this alert plugin params }
*/
List<PluginParams> getParams();
@Override
public TaskChannel create() {
return new ConditionsLogicTaskChannel();
}
} }

30
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannel.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
public class DependentLogicTaskChannel extends AbstractLogicTaskChannel {
@Override
public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(taskParams, DependentParameters.class);
}
}

38
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannelFactory.java

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import com.google.auto.service.AutoService;
@AutoService(TaskChannelFactory.class)
public class DependentLogicTaskChannelFactory implements TaskChannelFactory {
public static final String NAME = "DEPENDENT";
@Override
public String getName() {
return NAME;
}
@Override
public TaskChannel create() {
return new DependentLogicTaskChannel();
}
}

30
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannel.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters;
public class DynamicLogicTaskChannel extends AbstractLogicTaskChannel {
@Override
public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(taskParams, DynamicParameters.class);
}
}

38
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannelFactory.java

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import com.google.auto.service.AutoService;
@AutoService(TaskChannelFactory.class)
public class DynamicLogicTaskChannelFactory implements TaskChannelFactory {
public static final String NAME = "DYNAMIC";
@Override
public String getName() {
return NAME;
}
@Override
public TaskChannel create() {
return new DynamicLogicTaskChannel();
}
}

30
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannel.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters;
public class SubWorkflowLogicTaskChannel extends AbstractLogicTaskChannel {
@Override
public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(taskParams, SubProcessParameters.class);
}
}

39
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannelFactory.java

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import com.google.auto.service.AutoService;
@AutoService(TaskChannelFactory.class)
public class SubWorkflowLogicTaskChannelFactory implements TaskChannelFactory {
public static final String NAME = "SUB_PROCESS";
@Override
public String getName() {
return NAME;
}
@Override
public TaskChannel create() {
return new SubWorkflowLogicTaskChannel();
}
}

30
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannel.java

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
public class SwitchLogicTaskChannel extends AbstractLogicTaskChannel {
@Override
public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(taskParams, SwitchParameters.class);
}
}

39
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannelFactory.java

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.task;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import com.google.auto.service.AutoService;
@AutoService(TaskChannelFactory.class)
public class SwitchLogicTaskChannelFactory implements TaskChannelFactory {
public static final String NAME = "SWITCH";
@Override
public String getName() {
return NAME;
}
@Override
public TaskChannel create() {
return new SwitchLogicTaskChannel();
}
}

55
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskTypeUtils.java

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.utils;
import org.apache.dolphinscheduler.plugin.task.api.ILogicTaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.DynamicLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import lombok.experimental.UtilityClass;
@UtilityClass
public class TaskTypeUtils {
public boolean isSwitchTask(String taskType) {
return SwitchLogicTaskChannelFactory.NAME.equals(taskType);
}
public boolean isConditionTask(String taskType) {
return ConditionsLogicTaskChannelFactory.NAME.equals(taskType);
}
public boolean isSubWorkflowTask(String taskType) {
return SubWorkflowLogicTaskChannelFactory.NAME.equals(taskType);
}
public boolean isDependentTask(String taskType) {
return SubWorkflowLogicTaskChannelFactory.NAME.equals(taskType);
}
public boolean isDynamicTask(String taskType) {
return DynamicLogicTaskChannelFactory.NAME.equals(taskType);
}
public boolean isLogicTask(String taskType) {
return TaskPluginManager.getTaskChannel(taskType) instanceof ILogicTaskChannel;
}
}

50
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java

@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api.utils;
import java.util.Set;
import lombok.experimental.UtilityClass;
import com.google.common.collect.Sets;
@UtilityClass
public class TaskUtils {
private final String blockingLogicTask = "BLOCKING";
private final String conditionLogicTask = "CONDITIONS";
private final String dependentLogicTask = "DEPENDENT";
private final String subWorkflowLogicTask = "SUB_PROCESS";
private final String switchLogicTask = "SWITCH";
private final String dynamicLogicTask = "DYNAMIC";
// todo: Add to SPI
private final Set<String> MASTER_TASK_TYPES = Sets.newHashSet(
blockingLogicTask,
conditionLogicTask,
dependentLogicTask,
subWorkflowLogicTask,
switchLogicTask,
dynamicLogicTask);
// todo: add to task plugin spi
public boolean isLogicTask(String taskType) {
return MASTER_TASK_TYPES.contains(taskType);
}
}

44
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManagerTest.java

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.api;
import static com.google.common.truth.Truth.assertThat;
import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.DynamicLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
class TaskPluginManagerTest {
@ParameterizedTest
@ValueSource(strings = {
ConditionsLogicTaskChannelFactory.NAME,
DependentLogicTaskChannelFactory.NAME,
DynamicLogicTaskChannelFactory.NAME,
SubWorkflowLogicTaskChannelFactory.NAME,
SwitchLogicTaskChannelFactory.NAME})
void testGetTaskChannel_logicTaskChannel(String type) {
assertThat(TaskPluginManager.getTaskChannel(type)).isNotNull();
}
}

18
dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannel.java

@ -22,31 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
/**
* chunjun task channel
*/
public class ChunJunTaskChannel implements TaskChannel { public class ChunJunTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public AbstractTask createTask(TaskExecutionContext taskExecutionContext) { public AbstractTask createTask(TaskExecutionContext taskExecutionContext) {
return new ChunJunTask(taskExecutionContext); return new ChunJunTask(taskExecutionContext);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), ChunJunParameters.class); return JSONUtils.parseObject(taskParams, ChunJunParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return JSONUtils.parseObject(parameters, ChunJunParameters.class).getResources();
}
} }

12
dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannelFactory.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.chunjun;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -47,13 +44,4 @@ public class ChunJunTaskChannelFactory implements TaskChannelFactory {
return "CHUNJUN"; return "CHUNJUN";
} }
/**
* Returns the configurable parameters that this plugin needs to display on the web ui
*
* @return this plugin params
*/
@Override
public List<PluginParams> getParams() {
return null;
}
} }

14
dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannel.java

@ -21,27 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DatafactoryTaskChannel implements TaskChannel { public class DatafactoryTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public DatafactoryTask createTask(TaskExecutionContext taskRequest) { public DatafactoryTask createTask(TaskExecutionContext taskRequest) {
return new DatafactoryTask(taskRequest); return new DatafactoryTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DatafactoryParameters.class); return JSONUtils.parseObject(taskParams, DatafactoryParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
}
} }

8
dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannelFactory.java

@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.datafactory;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.Collections;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -39,8 +35,4 @@ public class DatafactoryTaskChannelFactory implements TaskChannelFactory {
return "DATA_FACTORY"; return "DATA_FACTORY";
} }
@Override
public List<PluginParams> getParams() {
return Collections.emptyList();
}
} }

15
dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java

@ -22,29 +22,18 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DataQualityTaskChannel implements TaskChannel { public class DataQualityTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public AbstractTask createTask(TaskExecutionContext taskRequest) { public AbstractTask createTask(TaskExecutionContext taskRequest) {
return new DataQualityTask(taskRequest); return new DataQualityTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DataQualityParameters.class); return JSONUtils.parseObject(taskParams, DataQualityParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
}
} }

8
dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannelFactory.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dq;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -33,11 +30,6 @@ public class DataQualityTaskChannelFactory implements TaskChannelFactory {
return "DATA_QUALITY"; return "DATA_QUALITY";
} }
@Override
public List<PluginParams> getParams() {
return null;
}
@Override @Override
public TaskChannel create() { public TaskChannel create() {
return new DataQualityTaskChannel(); return new DataQualityTaskChannel();

15
dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java

@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DatasyncTaskChannel implements TaskChannel { public class DatasyncTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public DatasyncTask createTask(TaskExecutionContext taskRequest) { public DatasyncTask createTask(TaskExecutionContext taskRequest) {
return new DatasyncTask(taskRequest); return new DatasyncTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DatasyncParameters.class); return JSONUtils.parseObject(taskParams, DatasyncParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
}
} }

8
dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java

@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.datasync;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.Collections;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -39,8 +35,4 @@ public class DatasyncTaskChannelFactory implements TaskChannelFactory {
return "DATASYNC"; return "DATASYNC";
} }
@Override
public List<PluginParams> getParams() {
return Collections.emptyList();
}
} }

15
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java

@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DataxTaskChannel implements TaskChannel { public class DataxTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public AbstractTask createTask(TaskExecutionContext taskRequest) { public AbstractTask createTask(TaskExecutionContext taskRequest) {
return new DataxTask(taskRequest); return new DataxTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DataxParameters.class); return JSONUtils.parseObject(taskParams, DataxParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return JSONUtils.parseObject(parameters, DataxParameters.class).getResources();
}
} }

8
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java

@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.datax;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -33,11 +30,6 @@ public class DataxTaskChannelFactory implements TaskChannelFactory {
return "DATAX"; return "DATAX";
} }
@Override
public List<PluginParams> getParams() {
return null;
}
@Override @Override
public TaskChannel create() { public TaskChannel create() {
return new DataxTaskChannel(); return new DataxTaskChannel();

15
dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java

@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DinkyTaskChannel implements TaskChannel { public class DinkyTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
// nothing to do
}
@Override @Override
public AbstractTask createTask(TaskExecutionContext taskRequest) { public AbstractTask createTask(TaskExecutionContext taskRequest) {
return new DinkyTask(taskRequest); return new DinkyTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DinkyParameters.class); return JSONUtils.parseObject(taskParams, DinkyParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
}
} }

9
dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java

@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dinky;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.ArrayList;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -34,11 +30,6 @@ public class DinkyTaskChannelFactory implements TaskChannelFactory {
return "DINKY"; return "DINKY";
} }
@Override
public List<PluginParams> getParams() {
return new ArrayList<>();
}
@Override @Override
public TaskChannel create() { public TaskChannel create() {
return new DinkyTaskChannel(); return new DinkyTaskChannel();

16
dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannel.java

@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DmsTaskChannel implements TaskChannel { public class DmsTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public DmsTask createTask(TaskExecutionContext taskRequest) { public DmsTask createTask(TaskExecutionContext taskRequest) {
return new DmsTask(taskRequest); return new DmsTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DmsParameters.class); return JSONUtils.parseObject(taskParams, DmsParameters.class);
}
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
} }
} }

8
dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java

@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dms;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.Collections;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -39,8 +35,4 @@ public class DmsTaskChannelFactory implements TaskChannelFactory {
return "DMS"; return "DMS";
} }
@Override
public List<PluginParams> getParams() {
return Collections.emptyList();
}
} }

16
dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java

@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class DvcTaskChannel implements TaskChannel { public class DvcTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public DvcTask createTask(TaskExecutionContext taskRequest) { public DvcTask createTask(TaskExecutionContext taskRequest) {
return new DvcTask(taskRequest); return new DvcTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), DvcParameters.class); return JSONUtils.parseObject(taskParams, DvcParameters.class);
}
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
} }
} }

23
dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java

@ -19,14 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dvc;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.ParamsOptions;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.spi.params.base.Validate;
import org.apache.dolphinscheduler.spi.params.input.InputParam;
import org.apache.dolphinscheduler.spi.params.radio.RadioParam;
import java.util.ArrayList;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -43,19 +35,4 @@ public class DvcTaskChannelFactory implements TaskChannelFactory {
return "DVC"; return "DVC";
} }
@Override
public List<PluginParams> getParams() {
List<PluginParams> paramsList = new ArrayList<>();
InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')")
.addValidate(Validate.newBuilder().setRequired(true).build()).build();
RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG")
.addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false))
.addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build();
paramsList.add(nodeName);
paramsList.add(runFlag);
return paramsList;
}
} }

15
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java

@ -22,16 +22,9 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
public class EmrTaskChannel implements TaskChannel { public class EmrTaskChannel implements TaskChannel {
@Override
public void cancelApplication(boolean status) {
// no need
}
@Override @Override
public AbstractTask createTask(TaskExecutionContext taskRequest) { public AbstractTask createTask(TaskExecutionContext taskRequest) {
EmrParameters emrParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), EmrParameters.class); EmrParameters emrParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), EmrParameters.class);
@ -46,12 +39,8 @@ public class EmrTaskChannel implements TaskChannel {
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), EmrParameters.class); return JSONUtils.parseObject(taskParams, EmrParameters.class);
} }
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
}
} }

9
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java

@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.emr;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.Collections;
import java.util.List;
import com.google.auto.service.AutoService; import com.google.auto.service.AutoService;
@ -34,11 +30,6 @@ public class EmrTaskChannelFactory implements TaskChannelFactory {
return "EMR"; return "EMR";
} }
@Override
public List<PluginParams> getParams() {
return Collections.emptyList();
}
@Override @Override
public TaskChannel create() { public TaskChannel create() {
return new EmrTaskChannel(); return new EmrTaskChannel();

26
dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java

@ -18,42 +18,20 @@
package org.apache.dolphinscheduler.plugin.task.flink; package org.apache.dolphinscheduler.plugin.task.flink;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTaskChannel; import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTaskChannel;
public class FlinkStreamTaskChannel implements StreamTaskChannel { public class FlinkStreamTaskChannel implements StreamTaskChannel {
@Override
public void cancelApplication(boolean status) {
}
@Override @Override
public FlinkStreamTask createTask(TaskExecutionContext taskRequest) { public FlinkStreamTask createTask(TaskExecutionContext taskRequest) {
return new FlinkStreamTask(taskRequest); return new FlinkStreamTask(taskRequest);
} }
@Override @Override
public AbstractParameters parseParameters(ParametersNode parametersNode) { public AbstractParameters parseParameters(String taskParams) {
return JSONUtils.parseObject(parametersNode.getTaskParams(), FlinkStreamParameters.class); return JSONUtils.parseObject(taskParams, FlinkStreamParameters.class);
}
@Override
public ResourceParametersHelper getResources(String parameters) {
return null;
}
@Override
public AbstractTask pauseTask(TaskExecutionContext taskExecutionContext) {
return null;
} }
@Override
public AbstractTask recoverTask(TaskExecutionContext taskExecutionContext) {
return null;
}
} }

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save