From ec6a3e0727214b31e13cc22cd945c336e2fe1b64 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Tue, 25 Jun 2024 13:32:13 +0800 Subject: [PATCH] [DSIP-46] Unify the logic task params data structure between front-end and back-end (#16193) --- .../api/ApiApplicationServer.java | 20 +- .../api/service/impl/ExecutorServiceImpl.java | 4 +- .../impl/ProcessDefinitionServiceImpl.java | 24 +- .../impl/ProcessInstanceServiceImpl.java | 20 +- .../impl/ProcessTaskRelationServiceImpl.java | 10 +- .../impl/TaskDefinitionServiceImpl.java | 30 +- .../impl/WorkFlowLineageServiceImpl.java | 17 +- .../service/ProcessInstanceServiceTest.java | 3 +- .../TaskDefinitionServiceImplTest.java | 22 +- .../common/constants/Constants.java | 2 - .../common/enums/StateEventType.java | 1 - .../common/enums/WorkflowExecutionStatus.java | 9 +- .../entity/DependentProcessDefinition.java | 9 +- .../dao/entity/ProcessInstance.java | 6 - .../dao/entity/TaskInstance.java | 133 +---- .../dao/utils/TaskInstanceUtils.java | 2 - .../dao/entity/TaskInstanceTest.java | 33 -- .../impl/ProcessInstanceDaoImplTest.java | 4 +- .../src/test/resources/logback.xml | 21 + .../server/master/MasterServer.java | 2 +- .../event/WorkflowBlockStateEventHandler.java | 59 --- .../runner/StreamTaskExecuteRunnable.java | 15 +- .../runner/TaskExecutionContextFactory.java | 13 +- .../runner/WorkflowExecuteRunnable.java | 79 +-- .../dispatcher/TaskDispatchFactory.java | 7 +- .../TaskExecuteRunnableOperatorManager.java | 11 +- .../master/runner/task/BaseSyncLogicTask.java | 15 +- .../task/blocking/BlockingLogicTask.java | 139 ------ .../BlockingLogicTaskPluginFactory.java | 51 -- .../task/condition/ConditionLogicTask.java | 51 +- .../ConditionLogicTaskPluginFactory.java | 21 +- .../DependentAsyncTaskExecuteFunction.java | 15 +- .../task/dependent/DependentLogicTask.java | 20 +- .../DependentLogicTaskPluginFactory.java | 9 +- .../subworkflow/SubWorkflowLogicTask.java | 13 +- .../SubWorkflowLogicTaskPluginFactory.java | 13 +- .../task/switchtask/SwitchLogicTask.java | 132 +++-- .../SwitchLogicTaskPluginFactory.java | 10 +- .../master/service/WorkerFailoverService.java | 4 +- .../server/master/utils/TaskUtils.java | 48 -- .../runner/WorkflowExecuteRunnableTest.java | 1 - .../dispatcher/TaskDispatchFactoryTest.java | 47 +- .../task/dynamic/DynamicLogicTaskTest.java | 10 +- .../master/service/FailoverServiceTest.java | 8 +- .../server/master/utils/TaskUtilsTest.java | 39 -- .../service/model/TaskNode.java | 129 ----- .../service/process/ProcessService.java | 2 - .../service/process/ProcessServiceImpl.java | 29 +- .../service/utils/DagHelper.java | 159 ++---- .../service/utils/DagHelperTest.java | 465 +++++++++++------- .../storage/hdfs/HdfsStorageOperator.java | 7 +- .../dolphinscheduler-task-api/pom.xml | 5 + .../plugin/task/api/ILogicTaskChannel.java | 21 +- .../plugin/task/api/TaskChannel.java | 10 +- .../plugin/task/api/TaskChannelFactory.java | 13 +- .../plugin/task/api/TaskConstants.java | 21 - .../plugin/task/api/TaskPluginManager.java | 96 ++-- .../task/api/model/DynamicInputParameter.java | 4 + .../plugin/task/api/model/SwitchResultVo.java | 15 +- .../api/parameters/BlockingParameters.java | 51 -- .../api/parameters/ConditionsParameters.java | 41 +- .../api/parameters/DependentParameters.java | 41 +- .../api/parameters/DynamicParameters.java | 6 + .../task/api/parameters/ParametersNode.java | 116 ----- .../api/parameters/SubProcessParameters.java | 5 +- .../task/api/parameters/SwitchParameters.java | 85 ++-- .../task/api/stream/StreamTaskChannel.java | 5 - .../api/task/AbstractLogicTaskChannel.java | 48 +- .../api/task/ConditionsLogicTaskChannel.java | 31 ++ .../ConditionsLogicTaskChannelFactory.java | 34 +- .../api/task/DependentLogicTaskChannel.java | 30 ++ .../DependentLogicTaskChannelFactory.java | 38 ++ .../api/task/DynamicLogicTaskChannel.java | 30 ++ .../task/DynamicLogicTaskChannelFactory.java | 38 ++ .../api/task/SubWorkflowLogicTaskChannel.java | 30 ++ .../SubWorkflowLogicTaskChannelFactory.java | 39 ++ .../task/api/task/SwitchLogicTaskChannel.java | 30 ++ .../task/SwitchLogicTaskChannelFactory.java | 39 ++ .../plugin/task/api/utils/TaskTypeUtils.java | 55 +++ .../plugin/task/api/utils/TaskUtils.java | 50 -- .../task/api/TaskPluginManagerTest.java | 44 ++ .../task/chunjun/ChunJunTaskChannel.java | 18 +- .../chunjun/ChunJunTaskChannelFactory.java | 12 - .../datafactory/DatafactoryTaskChannel.java | 14 +- .../DatafactoryTaskChannelFactory.java | 8 - .../task/dq/DataQualityTaskChannel.java | 15 +- .../dq/DataQualityTaskChannelFactory.java | 8 - .../task/datasync/DatasyncTaskChannel.java | 15 +- .../datasync/DatasyncTaskChannelFactory.java | 8 - .../plugin/task/datax/DataxTaskChannel.java | 15 +- .../task/datax/DataxTaskChannelFactory.java | 8 - .../plugin/task/dinky/DinkyTaskChannel.java | 15 +- .../task/dinky/DinkyTaskChannelFactory.java | 9 - .../plugin/task/dms/DmsTaskChannel.java | 16 +- .../task/dms/DmsTaskChannelFactory.java | 8 - .../plugin/task/dvc/DvcTaskChannel.java | 16 +- .../task/dvc/DvcTaskChannelFactory.java | 23 - .../plugin/task/emr/EmrTaskChannel.java | 15 +- .../task/emr/EmrTaskChannelFactory.java | 9 - .../task/flink/FlinkStreamTaskChannel.java | 26 +- .../flink/FlinkStreamTaskChannelFactory.java | 7 - .../plugin/task/flink/FlinkTaskChannel.java | 15 +- .../task/flink/FlinkTaskChannelFactory.java | 7 - .../task/hivecli/HiveCliTaskChannel.java | 15 +- .../hivecli/HiveCliTaskChannelFactory.java | 7 - .../plugin/task/http/HttpTaskChannel.java | 15 +- .../task/http/HttpTaskChannelFactory.java | 8 - .../plugin/task/java/JavaTaskChannel.java | 39 +- .../task/java/JavaTaskChannelFactory.java | 12 - .../task/jupyter/JupyterTaskChannel.java | 15 +- .../jupyter/JupyterTaskChannelFactory.java | 8 - .../plugin/task/k8s/K8sTaskChannel.java | 16 +- .../task/k8s/K8sTaskChannelFactory.java | 8 - .../plugin/kubeflow/KubeflowTaskChannel.java | 15 +- .../kubeflow/KubeflowTaskChannelFactory.java | 9 - .../plugin/task/linkis/LinkisTaskChannel.java | 16 +- .../task/linkis/LinkisTaskChannelFactory.java | 7 - .../plugin/task/mlflow/MlflowTaskChannel.java | 16 +- .../task/mlflow/MlflowTaskChannelFactory.java | 23 - .../plugin/task/mr/MapReduceTaskChannel.java | 15 +- .../task/mr/MapReduceTaskChannelFactory.java | 8 - .../task/openmldb/OpenmldbTaskChannel.java | 15 +- .../openmldb/OpenmldbTaskChannelFactory.java | 7 - .../plugin/task/pigeon/PigeonTaskChannel.java | 15 +- .../task/pigeon/PigeonTaskChannelFactory.java | 14 - .../task/procedure/ProcedureTaskChannel.java | 15 +- .../ProcedureTaskChannelFactory.java | 8 - .../plugin/task/python/PythonTaskChannel.java | 15 +- .../task/python/PythonTaskChannelFactory.java | 7 - .../task/pytorch/PytorchTaskChannel.java | 16 +- .../pytorch/PytorchTaskChannelFactory.java | 8 - .../remoteshell/RemoteShellTaskChannel.java | 15 +- .../RemoteShellTaskChannelFactory.java | 27 - .../task/sagemaker/SagemakerTaskChannel.java | 16 +- .../SagemakerTaskChannelFactory.java | 8 - .../task/seatunnel/SeatunnelTaskChannel.java | 16 +- .../SeatunnelTaskChannelFactory.java | 7 - .../plugin/task/shell/ShellTaskChannel.java | 16 +- .../task/shell/ShellTaskChannelFactory.java | 31 +- .../plugin/task/spark/SparkTaskChannel.java | 15 +- .../task/spark/SparkTaskChannelFactory.java | 8 - .../plugin/task/sql/SqlTaskChannel.java | 16 +- .../task/sql/SqlTaskChannelFactory.java | 12 +- .../plugin/task/sqoop/SqoopTaskChannel.java | 14 +- .../task/sqoop/SqoopTaskChannelFactory.java | 8 - .../task/zeppelin/ZeppelinTaskChannel.java | 15 +- .../zeppelin/ZeppelinTaskChannelFactory.java | 8 - .../v200/V200DolphinSchedulerUpgrader.java | 14 +- .../server/worker/WorkerServer.java | 2 +- .../worker/runner/WorkerTaskExecutor.java | 2 +- .../utils/TaskExecutionContextUtils.java | 7 +- 151 files changed, 1333 insertions(+), 2647 deletions(-) create mode 100644 dolphinscheduler-dao/src/test/resources/logback.xml delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/TaskUtils.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/TaskUtilsTest.java rename dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java => dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ILogicTaskChannel.java (70%) delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java rename dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java => dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/AbstractLogicTaskChannel.java (58%) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannel.java rename dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java => dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannelFactory.java (55%) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannel.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannelFactory.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskTypeUtils.java delete mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManagerTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index f8b705179a..6b183e369c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -19,22 +19,14 @@ package org.apache.dolphinscheduler.api; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; import org.apache.dolphinscheduler.common.CommonConfiguration; -import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; -import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; -import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import org.apache.dolphinscheduler.service.ServiceConfiguration; -import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -68,17 +60,7 @@ public class ApiApplicationServer { @EventListener public void run(ApplicationReadyEvent readyEvent) { log.info("Received spring application context ready event will load taskPlugin and write to DB"); - // install task plugin - TaskPluginManager.loadPlugin(); DataSourceProcessorProvider.initialize(); - for (Map.Entry entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) { - String taskPluginName = entry.getKey(); - TaskChannelFactory taskChannelFactory = entry.getValue(); - List params = taskChannelFactory.getParams(); - String paramsJson = PluginParamsTransfer.transferParamsToJson(params); - - PluginDefine pluginDefine = new PluginDefine(taskPluginName, PluginType.TASK.getDesc(), paramsJson); - pluginDao.addOrUpdatePluginDefine(pluginDefine); - } + TaskPluginManager.loadTaskPlugin(); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 3e613b11fa..6bb7d228ff 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -89,8 +89,8 @@ import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest; import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.cron.CronUtils; @@ -361,7 +361,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ // find out the process definition code Set processDefinitionCodeSet = new HashSet<>(); taskDefinitions.stream() - .filter(task -> TaskConstants.TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(task.getTaskType())).forEach( + .filter(task -> TaskTypeUtils.isSubWorkflowTask(task.getTaskType())).forEach( taskDefinition -> processDefinitionCodeSet.add(Long.valueOf( JSONUtils.getNodeString(taskDefinition.getTaskParams(), CMD_PARAM_SUB_PROCESS_DEFINE_CODE)))); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 6a4251cef1..820d35b3b1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -39,7 +39,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.IMPORT_SUFF import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOCAL_PARAMS_LIST; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL; +import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters; import org.apache.dolphinscheduler.api.dto.DagDataSchedule; import org.apache.dolphinscheduler.api.dto.treeview.Instance; @@ -109,12 +109,12 @@ import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao; import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionLogDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao; import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; +import org.apache.dolphinscheduler.plugin.task.sql.SqlTaskChannelFactory; import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -421,11 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson); } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskDefinitionLog.getTaskType()) - .taskParams(taskDefinitionLog.getTaskParams()) - .dependence(taskDefinitionLog.getDependence()) - .build())) { + if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) { log.error( "Generate task definition list failed, the given task definition parameter is invalided, taskName: {}, taskDefinition: {}", taskDefinitionLog.getName(), taskDefinitionLog); @@ -1386,7 +1382,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro sqlParameters.setLocalParams(Collections.emptyList()); taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters)); taskDefinition.setCode(CodeGenerateUtils.genCode()); - taskDefinition.setTaskType(TASK_TYPE_SQL); + taskDefinition.setTaskType(SqlTaskChannelFactory.NAME); taskDefinition.setFailRetryTimes(0); taskDefinition.setFailRetryInterval(0); taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE); @@ -1615,13 +1611,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // check whether the process definition json is normal for (TaskNode taskNode : taskNodes) { - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskNode.getType()) - .taskParams(taskNode.getTaskParams()) - .dependence(taskNode.getDependence()) - .switchResult(taskNode.getSwitchResult()) - .build())) { - log.error("Task node {} parameter invalid.", taskNode.getName()); + if (!checkTaskParameters(taskNode.getType(), taskNode.getParams())) { putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName()); return result; } @@ -1891,7 +1881,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro long subProcessCode = 0L; // if process is sub process, the return sub id, or sub id=0 - if (taskInstance.isSubProcess()) { + if (TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) { TaskDefinition taskDefinition = taskDefinitionMap.get(taskInstance.getTaskCode()); subProcessCode = Long.parseLong(JSONUtils.parseObject( taskDefinition.getTaskParams()).path(CMD_PARAM_SUB_PROCESS_DEFINE_CODE).asText()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 06e9fd95db..5e5d703017 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -26,8 +26,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.GLOBAL_PARA import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS; import static org.apache.dolphinscheduler.common.constants.Constants.PROCESS_INSTANCE_STATE; import static org.apache.dolphinscheduler.common.constants.Constants.TASK_LIST; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; +import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.dto.DynamicSubWorkflowDto; @@ -77,11 +76,10 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.WorkflowUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -498,7 +496,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId); } - if (!taskInstance.isDynamic()) { + if (!TaskTypeUtils.isDynamicTask(taskInstance.getTaskType())) { putMsg(result, Status.TASK_INSTANCE_NOT_DYNAMIC_TASK, taskInstance.getName()); throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId); } @@ -548,7 +546,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce */ private void addDependResultForTaskList(User loginUser, List taskInstanceList) throws IOException { for (TaskInstance taskInstance : taskInstanceList) { - if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) { + if (TaskTypeUtils.isDependentTask(taskInstance.getTaskType())) { log.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}", taskInstance.getTaskCode(), taskInstance.getId()); // TODO The result of dependent item should not be obtained from the log, waiting for optimization. @@ -628,9 +626,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce return result; } - if (!taskInstance.isSubProcess()) { - log.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.", - TASK_TYPE_SUB_PROCESS, projectCode, taskId); + if (!TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) { putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName()); return result; } @@ -714,11 +710,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskDefinitionLog.getTaskType()) - .taskParams(taskDefinitionLog.getTaskParams()) - .dependence(taskDefinitionLog.getDependence()) - .build())) { + if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) { log.error("Task parameters are invalid, taskDefinitionName:{}.", taskDefinitionLog.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName()); return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java index d8c568e50a..1f9fe6ff53 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java @@ -18,9 +18,6 @@ package org.apache.dolphinscheduler.api.service.impl; import static java.util.stream.Collectors.toSet; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationCreateRequest; import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationFilterRequest; @@ -46,6 +43,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.commons.collections4.CollectionUtils; @@ -354,9 +352,9 @@ public class ProcessTaskRelationServiceImpl extends BaseServiceImpl implements P } updateProcessDefiniteVersion(loginUser, result, processDefinition); updateRelation(loginUser, result, processDefinition, processTaskRelationList); - if (TASK_TYPE_CONDITIONS.equals(taskDefinition.getTaskType()) - || TASK_TYPE_DEPENDENT.equals(taskDefinition.getTaskType()) - || TASK_TYPE_SUB_PROCESS.equals(taskDefinition.getTaskType())) { + if (TaskTypeUtils.isConditionTask(taskDefinition.getTaskType()) + || TaskTypeUtils.isSubWorkflowTask(taskDefinition.getTaskType()) + || TaskTypeUtils.isDependentTask(taskDefinition.getTaskType())) { int deleteTaskDefinition = taskDefinitionMapper.deleteByCode(taskCode); if (0 == deleteTaskDefinition) { log.error("Delete task definition error, taskDefinitionCode:{}.", taskCode); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java index 2887606a1d..ae3410cb77 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java @@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_VERSION_VIEW; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_DEFINITION; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION; +import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters; import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest; import org.apache.dolphinscheduler.api.dto.task.TaskFilterRequest; @@ -67,8 +68,6 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.commons.collections4.CollectionUtils; @@ -167,11 +166,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe return result; } for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskDefinitionLog.getTaskType()) - .taskParams(taskDefinitionLog.getTaskParams()) - .dependence(taskDefinitionLog.getDependence()) - .build())) { + if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) { log.warn("Task definition {} parameters are invalid.", taskDefinitionLog.getName()); putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName()); return result; @@ -208,11 +203,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe Project project = projectMapper.queryByCode(taskDefinition.getProjectCode()); projectService.checkProjectAndAuthThrowException(user, project, permissions); - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskDefinition.getTaskType()) - .taskParams(taskDefinition.getTaskParams()) - .dependence(taskDefinition.getDependence()) - .build())) { + if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) { throw new ServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName()); } } @@ -321,12 +312,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return result; } - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskDefinition.getTaskType()) - .taskParams(taskDefinition.getTaskParams()) - .dependence(taskDefinition.getDependence()) - .build())) { - log.error("Task definition {} parameters are invalid", taskDefinition.getName()); + if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) { putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName()); return result; } @@ -732,13 +718,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj); return null; } - if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder() - .taskType(taskDefinitionToUpdate.getTaskType()) - .taskParams(taskDefinitionToUpdate.getTaskParams()) - .dependence(taskDefinitionToUpdate.getDependence()) - .build())) { - log.warn("Task definition parameters are invalid, taskDefinitionName:{}.", - taskDefinitionToUpdate.getName()); + if (!checkTaskParameters(taskDefinitionToUpdate.getTaskType(), taskDefinitionToUpdate.getTaskParams())) { putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName()); return null; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java index 014d22af57..9763f21b0f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.api.service.impl; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; - import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; @@ -39,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.commons.lang3.StringUtils; @@ -147,7 +146,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF List processDefinitionCodes) { for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) { for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters() - .getDependTaskList()) { + .getDependence().getDependTaskList()) { for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) { processDefinitionCodes.add(dependentItem.getDefinitionCode()); @@ -220,12 +219,12 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF List taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList); for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { if (taskDefinitionLog.getProjectCode() == projectCode) { - if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) { + if (TaskTypeUtils.isDependentTask(taskDefinitionLog.getTaskType())) { DependentParameters dependentParameters = JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); if (dependentParameters != null) { List dependTaskList = - dependentParameters.getDependTaskList(); + dependentParameters.getDependence().getDependTaskList(); if (!CollectionUtils.isEmpty(dependTaskList)) { for (DependentTaskModel taskModel : dependTaskList) { List dependItemList = taskModel.getDependItemList(); @@ -247,9 +246,9 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF /** * Query and return tasks dependence with string format, is a wrapper of queryTaskDepOnTask and task query method. * - * @param projectCode Project code want to query tasks dependence + * @param projectCode Project code want to query tasks dependence * @param processDefinitionCode Process definition code want to query tasks dependence - * @param taskCode Task code want to query tasks dependence + * @param taskCode Task code want to query tasks dependence * @return Optional of formatter message */ @Override @@ -271,7 +270,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF /** * Query tasks depend on process definition, include upstream or downstream * - * @param projectCode Project code want to query tasks dependence + * @param projectCode Project code want to query tasks dependence * @param processDefinitionCode Process definition code want to query tasks dependence * @return Set of TaskMainInfo */ @@ -291,7 +290,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF * Query downstream tasks depend on a process definition or a task * * @param processDefinitionCode Process definition code want to query tasks dependence - * @param taskCode Task code want to query tasks dependence + * @param taskCode Task code want to query tasks dependence * @return downstream dependent tasks */ @Override diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 7d9a4f9338..208d880fc1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -627,7 +627,8 @@ public class ProcessInstanceServiceTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(true); Map processInstanceFinishRes = processInstanceService.updateProcessInstance(loginUser, projectCode, 1, diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java index ff8ca6ba2c..8460c59924 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskDefinitionServiceImplTest.java @@ -64,6 +64,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessServiceImpl; @@ -165,7 +166,8 @@ public class TaskDefinitionServiceImplTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(true); Project project = getProject(); when(projectMapper.queryByCode(PROJECT_CODE)).thenReturn(project); @@ -194,7 +196,8 @@ public class TaskDefinitionServiceImplTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(true); String taskDefinitionJson = getTaskDefinitionJson(); @@ -441,7 +444,8 @@ public class TaskDefinitionServiceImplTest { () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); // error task definition - taskCreateRequest.setTaskParams(TASK_PARAMETER); + taskCreateRequest.setTaskType(ShellTaskChannelFactory.NAME); + taskCreateRequest.setTaskParams(JSONUtils.toJsonString(new HashMap<>())); doNothing().when(projectService).checkProjectAndAuthThrowException(user, getProject(), TASK_DEFINITION_CREATE); assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, () -> taskDefinitionService.createTaskDefinitionV2(user, taskCreateRequest)); @@ -449,7 +453,8 @@ public class TaskDefinitionServiceImplTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(true); // error create task definition object @@ -502,7 +507,8 @@ public class TaskDefinitionServiceImplTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(false); assertThrowsServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, () -> taskDefinitionService.updateTaskDefinitionV2(user, TASK_CODE, taskUpdateRequest)); @@ -511,7 +517,8 @@ public class TaskDefinitionServiceImplTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(true); // error task definition nothing update when(processService.isTaskOnline(TASK_CODE)).thenReturn(false); @@ -616,7 +623,8 @@ public class TaskDefinitionServiceImplTest { try ( MockedStatic taskPluginManagerMockedStatic = Mockito.mockStatic(TaskPluginManager.class)) { - taskPluginManagerMockedStatic.when(() -> TaskPluginManager.checkTaskParameters(Mockito.any())) + taskPluginManagerMockedStatic + .when(() -> TaskPluginManager.checkTaskParameters(Mockito.any(), Mockito.any())) .thenReturn(true); String taskDefinitionJson = getTaskDefinitionJson(); TaskDefinition taskDefinition = getTaskDefinition(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index fe0822927a..4800bff7f9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -272,8 +272,6 @@ public final class Constants { */ public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL"; - public static final String COMMON_TASK_TYPE = "common"; - public static final String DEFAULT = "default"; public static final String PASSWORD = "password"; public static final String XXXXXX = "******"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java index 6e25d2ed4b..220b56a939 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java @@ -27,7 +27,6 @@ public enum StateEventType { TASK_TIMEOUT(3, "task timeout"), WAKE_UP_TASK_GROUP(4, "wait task group"), TASK_RETRY(5, "task retry"), - PROCESS_BLOCKED(6, "process blocked"), PROCESS_SUBMIT_FAILED(7, "process submit failed"); StateEventType(int code, String descp) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java index 880db60257..923192ea7b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java @@ -38,8 +38,6 @@ public enum WorkflowExecutionStatus { SUCCESS(7, "success"), DELAY_EXECUTION(12, "delay execution"), SERIAL_WAIT(14, "serial wait"), - READY_BLOCK(15, "ready block"), - BLOCK(16, "block"), WAIT_TO_RUN(17, "wait to run"), ; @@ -59,7 +57,6 @@ public enum WorkflowExecutionStatus { READY_PAUSE.getCode(), READY_STOP.getCode(), SERIAL_WAIT.getCode(), - READY_BLOCK.getCode(), WAIT_TO_RUN.getCode() }; @@ -91,7 +88,7 @@ public enum WorkflowExecutionStatus { public boolean isFinished() { // todo: do we need to remove pause/block in finished judge? - return isSuccess() || isFailure() || isStop() || isPause() || isBlock(); + return isSuccess() || isFailure() || isStop() || isPause(); } /** @@ -119,10 +116,6 @@ public enum WorkflowExecutionStatus { return this == STOP; } - public boolean isBlock() { - return this == BLOCK; - } - public static int[] getNeedFailoverWorkflowInstanceState() { return NEED_FAILOVER_STATES; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java index 9fd77c944f..fcce2a33d3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.entity; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; @@ -67,7 +66,7 @@ public class DependentProcessDefinition { */ public CycleEnum getDependentCycle(long upstreamProcessDefinitionCode) { DependentParameters dependentParameters = this.getDependentParameters(); - List dependentTaskModelList = dependentParameters.getDependTaskList(); + List dependentTaskModelList = dependentParameters.getDependence().getDependTaskList(); for (DependentTaskModel dependentTaskModel : dependentTaskModelList) { List dependentItemList = dependentTaskModel.getDependItemList(); @@ -104,11 +103,7 @@ public class DependentProcessDefinition { } public DependentParameters getDependentParameters() { - return JSONUtils.parseObject(getDependence(), DependentParameters.class); - } - - public String getDependence() { - return JSONUtils.getNodeString(this.taskParams, Constants.DEPENDENCE); + return JSONUtils.parseObject(taskParams, DependentParameters.class); } public String getProcessDefinitionName() { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java index d8f01f473f..9ed8bca0ef 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java @@ -201,12 +201,6 @@ public class ProcessInstance { */ private Date restartTime; - /** - * workflow block flag - */ - @TableField(exist = false) - private boolean isBlocked; - /** * test flag */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 6fc14d5cac..629cfe9aea 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -17,30 +17,15 @@ package org.apache.dolphinscheduler.dao.entity; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DYNAMIC; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH; - -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; - -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import java.io.Serializable; import java.util.Date; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.TimeUnit; import lombok.Data; @@ -50,7 +35,6 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import com.fasterxml.jackson.core.type.TypeReference; /** * task instance @@ -190,21 +174,6 @@ public class TaskInstance implements Serializable { @TableField(updateStrategy = FieldStrategy.IGNORED) private String cacheKey; - /** - * dependency - */ - @TableField(exist = false) - private DependentParameters dependency; - - @TableField(exist = false) - private ConditionsParameters conditionsParameters; - - /** - * switch dependency - */ - @TableField(exist = false) - private SwitchParameters switchDependency; - /** * duration */ @@ -310,80 +279,6 @@ public class TaskInstance implements Serializable { this.executePath = executePath; } - public DependentParameters getDependency() { - if (this.dependency == null) { - Map taskParamsMap = - JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() { - }); - this.dependency = - JSONUtils.parseObject((String) taskParamsMap.get(Constants.DEPENDENCE), DependentParameters.class); - } - return this.dependency; - } - - public void setDependency(DependentParameters dependency) { - this.dependency = dependency; - } - - public ConditionsParameters getConditionsParameters() { - if (this.conditionsParameters == null) { - Map taskParamsMap = - JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() { - }); - this.conditionsParameters = - JSONUtils.parseObject((String) taskParamsMap.get(Constants.DEPENDENCE), ConditionsParameters.class); - } - return conditionsParameters; - } - - public ConditionsParameters.ConditionResult getConditionResult() { - Map taskParamsMap = - JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() { - }); - String conditionResult = (String) taskParamsMap.getOrDefault(Constants.CONDITION_RESULT, ""); - if (StringUtils.isNotEmpty(conditionResult)) { - return JSONUtils.parseObject(conditionResult, new TypeReference() { - }); - } - return null; - } - - public void setConditionResult(ConditionsParameters conditionsParameters) { - if (conditionsParameters == null) { - return; - } - Map taskParamsMap = - JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() { - }); - if (taskParamsMap == null) { - taskParamsMap = new HashMap<>(); - } - taskParamsMap.put(Constants.CONDITION_RESULT, JSONUtils.toJsonString(conditionsParameters)); - this.setTaskParams(JSONUtils.toJsonString(taskParamsMap)); - } - - public SwitchParameters getSwitchDependency() { - // todo: We need to directly use Jackson to deserialize the taskParam, rather than parse the map and get from - // field. - if (this.switchDependency == null) { - Map taskParamsMap = - JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() { - }); - this.switchDependency = - JSONUtils.parseObject((String) taskParamsMap.get(Constants.SWITCH_RESULT), SwitchParameters.class); - } - return this.switchDependency; - } - - public void setSwitchDependency(SwitchParameters switchDependency) { - Map taskParamsMap = - JSONUtils.parseObject(this.getTaskParams(), new TypeReference>() { - }); - taskParamsMap.put(Constants.SWITCH_RESULT, JSONUtils.toJsonString(switchDependency)); - this.switchDependency = switchDependency; - this.setTaskParams(JSONUtils.toJsonString(taskParamsMap)); - } - public boolean isTaskComplete() { return this.getState().isSuccess() @@ -392,30 +287,6 @@ public class TaskInstance implements Serializable { || this.getState().isForceSuccess(); } - public boolean isSubProcess() { - return TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(this.taskType); - } - - public boolean isDependTask() { - return TASK_TYPE_DEPENDENT.equalsIgnoreCase(this.taskType); - } - - public boolean isDynamic() { - return TASK_TYPE_DYNAMIC.equalsIgnoreCase(this.taskType); - } - - public boolean isConditionsTask() { - return TASK_TYPE_CONDITIONS.equalsIgnoreCase(this.taskType); - } - - public boolean isSwitchTask() { - return TASK_TYPE_SWITCH.equalsIgnoreCase(this.taskType); - } - - public boolean isBlockingTask() { - return TASK_TYPE_BLOCKING.equalsIgnoreCase(this.taskType); - } - public boolean isFirstRun() { return endTime == null; } @@ -427,7 +298,7 @@ public class TaskInstance implements Serializable { * @return can try result */ public boolean taskCanRetry() { - if (this.isSubProcess()) { + if (TaskTypeUtils.isSubWorkflowTask(getTaskType())) { return false; } if (this.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java index f75a2f5e11..e23fcb97b5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java @@ -55,11 +55,9 @@ public class TaskInstanceUtils { target.setPid(source.getPid()); target.setAppLink(source.getAppLink()); target.setFlag(source.getFlag()); - target.setDependency(source.getDependency()); // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on // task params, this is really a very bad practice. target.setTaskParams(source.getTaskParams()); - target.setSwitchDependency(source.getSwitchDependency()); target.setDuration(source.getDuration()); target.setMaxRetryTimes(source.getMaxRetryTimes()); target.setRetryInterval(source.getRetryInterval()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java index cb68cf7494..1bcc784cbc 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java @@ -17,12 +17,7 @@ package org.apache.dolphinscheduler.dao.entity; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; - import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; @@ -30,35 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameter import java.util.ArrayList; import java.util.List; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class TaskInstanceTest { - /** - * task instance sub process - */ - @Test - public void testTaskInstanceIsSubProcess() { - TaskInstance taskInstance = new TaskInstance(); - - // sub process - taskInstance.setTaskType(TASK_TYPE_SUB_PROCESS); - Assertions.assertTrue(taskInstance.isSubProcess()); - - // not sub process - taskInstance.setTaskType("HTTP"); - Assertions.assertFalse(taskInstance.isSubProcess()); - - // sub process - taskInstance.setTaskType(TASK_TYPE_CONDITIONS); - Assertions.assertTrue(taskInstance.isConditionsTask()); - - // sub process - taskInstance.setTaskType(TASK_TYPE_DEPENDENT); - Assertions.assertTrue(taskInstance.isDependTask()); - } - /** * test for TaskInstance.getDependence */ @@ -66,7 +36,6 @@ public class TaskInstanceTest { public void testTaskInstanceGetDependence() { TaskInstance taskInstance = new TaskInstance(); taskInstance.setTaskParams(JSONUtils.toJsonString(getDependentParameters())); - taskInstance.getDependency(); } /** @@ -82,8 +51,6 @@ public class TaskInstanceTest { dependentItem.setDefinitionCode(222L); dependentItem.setCycle("today"); dependentItems.add(dependentItem); - dependentParameters.setDependTaskList(dependTaskList); - dependentParameters.setRelation(DependentRelation.AND); return dependentParameters; } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java index 421309130f..9f69b7d90f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/ProcessInstanceDaoImplTest.java @@ -65,11 +65,9 @@ class ProcessInstanceDaoImplTest extends BaseDaoTest { WorkflowExecutionStatus.READY_STOP)); processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion, WorkflowExecutionStatus.SERIAL_WAIT)); - processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion, - WorkflowExecutionStatus.READY_BLOCK)); processInstanceDao.insert(createWorkflowInstance(workflowDefinitionCode, workflowDefinitionVersion, WorkflowExecutionStatus.WAIT_TO_RUN)); - assertEquals(8, processInstanceDao + assertEquals(7, processInstanceDao .queryByWorkflowCodeVersionStatus(workflowDefinitionCode, workflowDefinitionVersion, status).size()); } diff --git a/dolphinscheduler-dao/src/test/resources/logback.xml b/dolphinscheduler-dao/src/test/resources/logback.xml new file mode 100644 index 0000000000..9a182a18ef --- /dev/null +++ b/dolphinscheduler-dao/src/test/resources/logback.xml @@ -0,0 +1,21 @@ + + + + + + diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 9b507500b9..92d7dcfaec 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -108,7 +108,7 @@ public class MasterServer implements IStoppable { this.masterRPCServer.start(); // install task plugin - TaskPluginManager.loadPlugin(); + TaskPluginManager.loadTaskPlugin(); DataSourceProcessorProvider.initialize(); this.masterSlotManager.start(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java deleted file mode 100644 index 07f89fc544..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.event; - -import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; - -import java.util.Optional; - -import lombok.extern.slf4j.Slf4j; - -import com.google.auto.service.AutoService; - -@AutoService(StateEventHandler.class) -@Slf4j -public class WorkflowBlockStateEventHandler implements StateEventHandler { - - @Override - public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, - StateEvent stateEvent) throws StateEventHandleError { - log.info("Handle workflow instance state block event"); - Optional taskInstanceOptional = - workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId()); - if (!taskInstanceOptional.isPresent()) { - throw new StateEventHandleError("Cannot find taskInstance from taskMap by taskInstanceId: " - + stateEvent.getTaskInstanceId()); - } - TaskInstance task = taskInstanceOptional.get(); - - BlockingParameters parameters = JSONUtils.parseObject(task.getTaskParams(), BlockingParameters.class); - if (parameters != null && parameters.isAlertWhenBlocking()) { - workflowExecuteRunnable.processBlock(); - } - return true; - } - - @Override - public StateEventType getEventType() { - return StateEventType.PROCESS_BLOCKED; - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java index b09d9f1b1a..90fb25fcf0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java @@ -43,7 +43,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; @@ -89,8 +88,6 @@ public class StreamTaskExecuteRunnable implements Runnable { protected ProcessTaskRelationMapper processTaskRelationMapper; - protected TaskPluginManager taskPluginManager; - private StreamTaskInstanceExecCacheManager streamTaskInstanceExecCacheManager; protected TaskDefinition taskDefinition; @@ -115,7 +112,6 @@ public class StreamTaskExecuteRunnable implements Runnable { this.processService = SpringApplicationContext.getBean(ProcessService.class); this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class); this.workerTaskDispatcher = SpringApplicationContext.getBean(WorkerTaskDispatcher.class); - this.taskPluginManager = SpringApplicationContext.getBean(TaskPluginManager.class); this.processTaskRelationMapper = SpringApplicationContext.getBean(ProcessTaskRelationMapper.class); this.taskInstanceDao = SpringApplicationContext.getBean(TaskInstanceDao.class); this.streamTaskInstanceExecCacheManager = @@ -311,14 +307,11 @@ public class StreamTaskExecuteRunnable implements Runnable { return null; } - TaskChannel taskChannel = taskPluginManager.getTaskChannel(taskInstance.getTaskType()); - ResourceParametersHelper resources = taskChannel.getResources(taskInstance.getTaskParams()); + TaskChannel taskChannel = TaskPluginManager.getTaskChannel(taskInstance.getTaskType()); + ResourceParametersHelper resources = taskChannel.parseParameters(taskInstance.getTaskParams()).getResources(); - AbstractParameters baseParam = taskPluginManager.getParameters( - ParametersNode.builder() - .taskType(taskInstance.getTaskType()) - .taskParams(taskInstance.getTaskParams()) - .build()); + AbstractParameters baseParam = + TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams()); Map propertyMap = paramParsingPreparation(taskInstance, baseParam); TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() .buildWorkflowInstanceHost(masterConfig.getMasterAddress()) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index b3b0cf67ea..da4f5308d0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -49,7 +49,6 @@ import org.apache.dolphinscheduler.plugin.task.api.model.JdbcInfo; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.AbstractResourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; @@ -74,7 +73,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; @@ -102,16 +100,15 @@ public class TaskExecutionContextFactory { public TaskExecutionContext createTaskExecutionContext(TaskInstance taskInstance) throws TaskExecutionContextCreateException { ProcessInstance workflowInstance = taskInstance.getProcessInstance(); - ResourceParametersHelper resources = - Optional.ofNullable(TaskPluginManager.getTaskChannel(taskInstance.getTaskType())) - .map(taskChannel -> taskChannel.getResources(taskInstance.getTaskParams())) - .orElse(null); + ResourceParametersHelper resources = TaskPluginManager.getTaskChannel(taskInstance.getTaskType()) + .parseParameters(taskInstance.getTaskParams()) + .getResources(); setTaskResourceInfo(resources); Map businessParamsMap = curingParamsService.preBuildBusinessParams(workflowInstance); - AbstractParameters baseParam = TaskPluginManager.getParameters(ParametersNode.builder() - .taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build()); + AbstractParameters baseParam = + TaskPluginManager.parseTaskParameters(taskInstance.getTaskType(), taskInstance.getTaskParams()); Map propertyMap = curingParamsService.paramParsingPreparation(taskInstance, baseParam, workflowInstance); TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index 086a1b70a3..c58d741d41 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -28,7 +28,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.COMMA; import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM_DD_HH_MM_SS; import static org.apache.dolphinscheduler.dao.utils.EnvironmentUtils.getEnvironmentCodeOrDefault; import static org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils.getWorkerGroupOrDefault; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -61,8 +60,8 @@ import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHost import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.VarPoolUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; @@ -76,7 +75,6 @@ import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory; import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator; -import org.apache.dolphinscheduler.server.master.utils.TaskUtils; import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils; import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager; import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; @@ -300,7 +298,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { stateEvent, stateEventHandleException); ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); - } catch (Exception e) { + } catch (Throwable e) { // we catch the exception here, since if the state event handle failed, the state event will still // keep // in the stateEvents queue. @@ -381,9 +379,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { if (taskInstance.getIsCache().equals(Flag.YES)) { saveCacheTaskInstance(taskInstance); } - if (!workflowInstance.isBlocked()) { - submitPostNode(taskInstance.getTaskCode()); - } + submitPostNode(taskInstance.getTaskCode()); } else if (taskInstance.taskCanRetry() && !workflowInstance.getState().isReadyStop()) { // retry task log.info("Retry taskInstance taskInstance state: {}", taskInstance.getState()); @@ -950,16 +946,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { if (!taskInstance.getState().isFinished()) { taskExecuteRunnable.dispatch(); } else { - if (workflowExecuteContext.getWorkflowInstance().isBlocked()) { - TaskStateEvent processBlockEvent = TaskStateEvent.builder() - .processInstanceId(workflowExecuteContext.getWorkflowInstance().getId()) - .taskInstanceId(taskInstance.getId()) - .status(taskInstance.getState()) - .type(StateEventType.PROCESS_BLOCKED) - .build(); - this.stateEvents.add(processBlockEvent); - } - TaskStateEvent taskStateChangeEvent = TaskStateEvent.builder() .processInstanceId(workflowExecuteContext.getWorkflowInstance().getId()) .taskInstanceId(taskInstance.getId()) @@ -1096,7 +1082,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { taskInstance.setRetryInterval(taskNode.getRetryInterval()); // set task param - taskInstance.setTaskParams(taskNode.getTaskParams()); + taskInstance.setTaskParams(taskNode.getParams()); // set task group and priority taskInstance.setTaskGroupId(taskNode.getTaskGroupId()); @@ -1275,7 +1261,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { private boolean tryToTakeOverTaskInstance(TaskInstance taskInstance) { ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance(); - if (TaskUtils.isMasterTask(taskInstance.getTaskType())) { + if (TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) { return false; } try { @@ -1341,13 +1327,14 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { if (depTaskState.isKill()) { return DependResult.NON_EXEC; } - // ignore task state if current task is block - if (taskNode.isBlockingTask()) { + + // always return success if current task is condition + if (TaskTypeUtils.isConditionTask(taskNode.getType())) { continue; } - // always return success if current task is condition - if (taskNode.isConditionsTask()) { + // always return success if current task is switch + if (TaskTypeUtils.isSwitchTask(taskNode.getType())) { continue; } @@ -1392,7 +1379,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { private boolean dependTaskSuccess(Long dependNodeCode, Long nextNodeCode) { DAG dag = workflowExecuteContext.getWorkflowGraph().getDag(); TaskNode dependentNode = dag.getNode(dependNodeCode); - if (dependentNode.isConditionsTask()) { + if (TaskTypeUtils.isConditionTask(dependentNode.getType())) { // condition task need check the branch to run List nextTaskList = DagHelper.parseConditionTask(dependNodeCode, skipTaskNodeMap, dag, getCompleteTaskInstanceMap()); @@ -1405,12 +1392,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } return true; } - if (dependentNode.isSwitchTask()) { - TaskInstance dependentTaskInstance = taskInstanceMap.get(validTaskMap.get(dependentNode.getCode())); - SwitchParameters switchParameters = dependentTaskInstance.getSwitchDependency(); - return switchParameters.getDependTaskList().get(switchParameters.getResultConditionLocation()).getNextNode() - .contains(nextNodeCode); - } Optional existTaskInstanceOptional = getTaskInstance(dependNodeCode); if (!existTaskInstanceOptional.isPresent()) { return false; @@ -1450,8 +1431,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { */ private WorkflowExecutionStatus runningState(WorkflowExecutionStatus state) { if (state == WorkflowExecutionStatus.READY_STOP || state == WorkflowExecutionStatus.READY_PAUSE - || state == WorkflowExecutionStatus.READY_BLOCK || - state == WorkflowExecutionStatus.DELAY_EXECUTION) { + || state == WorkflowExecutionStatus.DELAY_EXECUTION) { // if the running task is not completed, the state remains unchanged return state; } else { @@ -1510,7 +1490,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } List pauseList = getCompleteTaskByState(TaskExecutionStatus.PAUSE); - if (CollectionUtils.isNotEmpty(pauseList) || workflowInstance.isBlocked() || !isComplementEnd() + if (CollectionUtils.isNotEmpty(pauseList) || !isComplementEnd() || standByTaskInstancePriorityQueue.size() > 0) { return WorkflowExecutionStatus.PAUSE; } else { @@ -1518,30 +1498,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } } - /** - * prepare for block - * if process has tasks still running, pause them - * if readyToSubmitTaskQueue is not empty, kill them - * else return block status directly - * - * @return ExecutionStatus - */ - private WorkflowExecutionStatus processReadyBlock() { - if (taskExecuteRunnableMap.size() > 0) { - for (DefaultTaskExecuteRunnable taskExecuteRunnable : taskExecuteRunnableMap.values()) { - if (!TASK_TYPE_BLOCKING.equals(taskExecuteRunnable.getTaskInstance().getTaskType())) { - taskExecuteRunnable.pause(); - } - } - } - if (standByTaskInstancePriorityQueue.size() > 0) { - for (Iterator iter = standByTaskInstancePriorityQueue.iterator(); iter.hasNext();) { - iter.next().setState(TaskExecutionStatus.PAUSE); - } - } - return WorkflowExecutionStatus.BLOCK; - } - /** * generate the latest process instance status by the tasks state * @@ -1557,13 +1513,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { return executionStatus; } - // block - if (state == WorkflowExecutionStatus.READY_BLOCK) { - WorkflowExecutionStatus executionStatus = processReadyBlock(); - log.info("The workflowInstance is ready to block, the workflowInstance status is {}", executionStatus); - return executionStatus; - } - // pause if (state == WorkflowExecutionStatus.READY_PAUSE) { WorkflowExecutionStatus executionStatus = processReadyPause(); @@ -2038,7 +1987,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } private void sendTaskLogOnMasterToRemoteIfNeeded(TaskInstance taskInstance) { - if (RemoteLogUtils.isRemoteLoggingEnable() && TaskUtils.isMasterTask(taskInstance.getTaskType())) { + if (RemoteLogUtils.isRemoteLoggingEnable() && TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) { RemoteLogUtils.sendRemoteLog(taskInstance.getLogPath()); log.info("Master sends task log {} to remote storage asynchronously.", taskInstance.getLogPath()); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java index 52469fb54c..9943ee1b5b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.master.runner.dispatcher; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.master.utils.TaskUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import lombok.extern.slf4j.Slf4j; @@ -36,7 +36,10 @@ public class TaskDispatchFactory { private WorkerTaskDispatcher workerTaskDispatcher; public TaskDispatcher getTaskDispatcher(String taskType) { - return TaskUtils.isMasterTask(taskType) ? masterTaskDispatcher : workerTaskDispatcher; + if (TaskTypeUtils.isLogicTask(taskType)) { + return masterTaskDispatcher; + } + return workerTaskDispatcher; } public TaskDispatcher getTaskDispatcher(TaskInstance taskInstance) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java index 1b92f5e75c..0a67e801dc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.server.master.runner.operator; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.utils.TaskUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -51,28 +51,29 @@ public class TaskExecuteRunnableOperatorManager { private LogicTaskExecuteRunnableTimeoutOperator logicTaskTimeoutOperator; public TaskExecuteRunnableOperator getTaskKillOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { - if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { return logicTaskKillOperator; } return taskKillOperator; } public TaskExecuteRunnableOperator getTaskPauseOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { - if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + return logicTaskPauseOperator; } return taskPauseOperator; } public TaskExecuteRunnableOperator getTaskDispatchOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { - if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { return logicTaskDispatchOperator; } return taskDispatchOperator; } public TaskExecuteRunnableOperator getTaskTimeoutOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { - if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + if (TaskTypeUtils.isLogicTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { return logicTaskTimeoutOperator; } return taskTimeoutOperator; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java index 10a4ec1e7c..064e054ed3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java @@ -18,9 +18,12 @@ package org.apache.dolphinscheduler.server.master.runner.task; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; +import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import lombok.extern.slf4j.Slf4j; @@ -28,10 +31,20 @@ import lombok.extern.slf4j.Slf4j; public abstract class BaseSyncLogicTask implements ISyncLogicTask { protected final TaskExecutionContext taskExecutionContext; + + protected final WorkflowExecuteRunnable workflowExecuteRunnable; + protected final TaskInstance taskInstance; protected final T taskParameters; - protected BaseSyncLogicTask(TaskExecutionContext taskExecutionContext, T taskParameters) { + protected BaseSyncLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable, + TaskExecutionContext taskExecutionContext, + T taskParameters) throws LogicTaskInitializeException { this.taskExecutionContext = taskExecutionContext; + this.workflowExecuteRunnable = workflowExecuteRunnable; + this.taskInstance = + workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId()).orElseThrow( + () -> new LogicTaskInitializeException( + "Cannot find the task instance in workflow execute runnable")); this.taskParameters = taskParameters; log.info("Success initialize task parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters)); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java deleted file mode 100644 index acc05aaf2d..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTask.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.runner.task.blocking; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.BlockingOpportunity; -import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; -import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; -import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; -import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils; -import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; -import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; -import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import lombok.extern.slf4j.Slf4j; - -import com.fasterxml.jackson.core.type.TypeReference; - -@Slf4j -public class BlockingLogicTask extends BaseSyncLogicTask { - - public static final String TASK_TYPE = "BLOCKING"; - - private final ProcessInstanceExecCacheManager processInstanceExecCacheManager; - - private final ProcessInstanceDao processInstanceDao; - - private final TaskInstanceDao taskInstanceDao; - - public BlockingLogicTask(TaskExecutionContext taskExecutionContext, - ProcessInstanceExecCacheManager processInstanceExecCacheManager, - ProcessInstanceDao processInstanceDao, - TaskInstanceDao taskInstanceDao) { - super(taskExecutionContext, - JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() { - })); - this.processInstanceExecCacheManager = processInstanceExecCacheManager; - this.processInstanceDao = processInstanceDao; - this.taskInstanceDao = taskInstanceDao; - } - - @Override - public void handle() throws MasterTaskExecuteException { - DependResult conditionResult = calculateConditionResult(); - DependResult expected = taskParameters.getBlockingOpportunity() - .equals(BlockingOpportunity.BLOCKING_ON_SUCCESS.getDesc()) - ? DependResult.SUCCESS - : DependResult.FAILED; - boolean isBlocked = (expected == conditionResult); - log.info("blocking opportunity: expected-->{}, actual-->{}", expected, conditionResult); - ProcessInstance workflowInstance = processInstanceExecCacheManager - .getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()).getWorkflowExecuteContext() - .getWorkflowInstance(); - workflowInstance.setBlocked(isBlocked); - if (isBlocked) { - workflowInstance.setStateWithDesc(WorkflowExecutionStatus.READY_BLOCK, "ready block"); - } - taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); - } - - private DependResult calculateConditionResult() throws MasterTaskExecuteException { - // todo: Directly get the task instance from the cache - Map completeTaskList = taskInstanceDao - .queryValidTaskListByWorkflowInstanceId(taskExecutionContext.getProcessInstanceId(), - taskExecutionContext.getTestFlag()) - .stream() - .collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity())); - - // todo: we need to parse the task parameter from TaskExecutionContext - TaskInstance taskInstance = - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) - .getTaskInstance(taskExecutionContext.getTaskInstanceId()) - .orElseThrow(() -> new MasterTaskExecuteException("Task instance not found")); - DependentParameters dependentParameters = taskInstance.getDependency(); - - List tempResultList = new ArrayList<>(); - for (DependentTaskModel dependentTaskModel : dependentParameters.getDependTaskList()) { - List itemDependResult = new ArrayList<>(); - for (DependentItem item : dependentTaskModel.getDependItemList()) { - itemDependResult.add(getDependResultForItem(item, completeTaskList)); - } - DependResult tempResult = - DependentUtils.getDependResultForRelation(dependentTaskModel.getRelation(), itemDependResult); - tempResultList.add(tempResult); - } - return DependentUtils.getDependResultForRelation(dependentParameters.getRelation(), tempResultList); - } - - private DependResult getDependResultForItem(DependentItem item, Map completeTaskList) { - - DependResult dependResult = DependResult.SUCCESS; - if (!completeTaskList.containsKey(item.getDepTaskCode())) { - log.info("depend item: {} have not completed yet.", item.getDepTaskCode()); - dependResult = DependResult.FAILED; - return dependResult; - } - TaskInstance taskInstance = completeTaskList.get(item.getDepTaskCode()); - if (taskInstance.getState() != item.getStatus()) { - log.info("depend item : {} expect status: {}, actual status: {}", item.getDepTaskCode(), item.getStatus(), - taskInstance.getState().name()); - dependResult = DependResult.FAILED; - } - log.info("Dependent item complete {} {},{}", - Constants.DEPENDENT_SPLIT, item.getDepTaskCode(), dependResult); - return dependResult; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java deleted file mode 100644 index b4fdd56c10..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/blocking/BlockingLogicTaskPluginFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.runner.task.blocking; - -import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; -import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class BlockingLogicTaskPluginFactory implements ILogicTaskPluginFactory { - - @Autowired - private ProcessInstanceDao processInstanceDao; - - @Autowired - private TaskInstanceDao taskInstanceDao; - - @Autowired - private ProcessInstanceExecCacheManager processInstanceExecCacheManager; - - @Override - public BlockingLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) { - return new BlockingLogicTask(taskExecutionContext, processInstanceExecCacheManager, processInstanceDao, - taskInstanceDao); - } - - @Override - public String getTaskType() { - return BlockingLogicTask.TASK_TYPE; - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java index b54f10d48d..10f7c52cf5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java @@ -17,9 +17,8 @@ package org.apache.dolphinscheduler.server.master.runner.task.condition; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; @@ -27,6 +26,8 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils; +import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; +import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask; import java.util.List; @@ -37,47 +38,42 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import com.fasterxml.jackson.core.type.TypeReference; + @Slf4j public class ConditionLogicTask extends BaseSyncLogicTask { public static final String TASK_TYPE = "CONDITIONS"; private final TaskInstanceDao taskInstanceDao; - private final ProcessInstanceDao workflowInstanceDao; - - private final TaskInstance taskInstance; - public ConditionLogicTask(TaskExecutionContext taskExecutionContext, - TaskInstance taskInstance, - TaskInstanceDao taskInstanceDao, - ProcessInstanceDao workflowInstanceDao) { - // todo: we need to change the parameter in front-end, so that we can directly use json to parse - super(taskExecutionContext, taskInstance.getConditionsParameters()); + public ConditionLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable, + TaskExecutionContext taskExecutionContext, + TaskInstanceDao taskInstanceDao) throws LogicTaskInitializeException { + super(workflowExecuteRunnable, taskExecutionContext, + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() { + })); this.taskInstanceDao = taskInstanceDao; - this.workflowInstanceDao = workflowInstanceDao; - this.taskInstance = taskInstance; } @Override public void handle() { - // calculate the conditionResult DependResult conditionResult = calculateConditionResult(); log.info("The condition result is {}", conditionResult); - taskParameters.setConditionSuccess(conditionResult == DependResult.SUCCESS); - taskInstance.setConditionsParameters(taskParameters); + taskParameters.getConditionResult().setConditionSuccess(conditionResult == DependResult.SUCCESS); + taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters)); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); } private DependResult calculateConditionResult() { - final ProcessInstance processInstance = - workflowInstanceDao.queryById(taskExecutionContext.getProcessInstanceId()); - final List taskInstances = - taskInstanceDao.queryValidTaskListByWorkflowInstanceId(processInstance.getId(), - processInstance.getTestFlag()); - final Map taskInstanceMap = - taskInstances.stream().collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity())); + final List taskInstances = taskInstanceDao.queryValidTaskListByWorkflowInstanceId( + taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTestFlag()); + final Map taskInstanceMap = taskInstances.stream() + .collect(Collectors.toMap(TaskInstance::getTaskCode, Function.identity())); - List dependResults = taskParameters.getDependTaskList().stream() + ConditionsParameters.ConditionDependency dependence = taskParameters.getDependence(); + List dependResults = dependence.getDependTaskList() + .stream() .map(dependentTaskModel -> DependentUtils.getDependResultForRelation( dependentTaskModel.getRelation(), dependentTaskModel.getDependItemList() @@ -85,7 +81,7 @@ public class ConditionLogicTask extends BaseSyncLogicTask .map(dependentItem -> getDependResultForItem(dependentItem, taskInstanceMap)) .collect(Collectors.toList()))) .collect(Collectors.toList()); - return DependentUtils.getDependResultForRelation(taskParameters.getRelation(), dependResults); + return DependentUtils.getDependResultForRelation(dependence.getRelation(), dependResults); } private DependResult getDependResultForItem(DependentItem item, Map taskInstanceMap) { @@ -96,8 +92,9 @@ public class ConditionLogicTask extends BaseSyncLogicTask return DependResult.FAILED; } - DependResult dependResult = - Objects.equals(item.getStatus(), taskInstance.getState()) ? DependResult.SUCCESS : DependResult.FAILED; + DependResult dependResult = Objects.equals(item.getStatus(), taskInstance.getState()) + ? DependResult.SUCCESS + : DependResult.FAILED; log.info("The depend item: {}", item); log.info("Expect status: {}", item.getStatus()); log.info("Actual status: {}", taskInstance.getState()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java index 4aee27f36d..f090b6d290 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.server.master.runner.task.condition; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; +import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import lombok.extern.slf4j.Slf4j; @@ -36,28 +36,19 @@ public class ConditionLogicTaskPluginFactory implements ILogicTaskPluginFactory< @Autowired private TaskInstanceDao taskInstanceDao; - @Autowired - private ProcessInstanceDao processInstanceDao; @Autowired private ProcessInstanceExecCacheManager processInstanceExecCacheManager; @Override public ConditionLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException { - TaskInstance taskInstance = - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) - .getTaskInstance(taskExecutionContext.getTaskInstanceId()) - .orElseThrow(() -> new LogicTaskInitializeException( - "Cannot find the task instance in workflow execute runnable")); - return new ConditionLogicTask( - taskExecutionContext, - taskInstance, - taskInstanceDao, - processInstanceDao); + WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId( + taskExecutionContext.getProcessInstanceId()); + return new ConditionLogicTask(workflowExecuteRunnable, taskExecutionContext, taskInstanceDao); } @Override public String getTaskType() { - return ConditionLogicTask.TASK_TYPE; + return ConditionsLogicTaskChannelFactory.NAME; } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java index 57f356e2b5..c3bd085657 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java @@ -122,10 +122,12 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti private List initializeDependentTaskList() { log.info("Begin to initialize dependent task list"); + List dependTaskList = dependentParameters.getDependence().getDependTaskList(); + final Set projectCodes = new HashSet<>(); final Set processDefinitionCodes = new HashSet<>(); final Set taskDefinitionCodes = new HashSet<>(); - for (DependentTaskModel taskModel : dependentParameters.getDependTaskList()) { + for (DependentTaskModel taskModel : dependTaskList) { for (DependentItem dependentItem : taskModel.getDependItemList()) { projectCodes.add(dependentItem.getProjectCode()); processDefinitionCodes.add(dependentItem.getDefinitionCode()); @@ -142,7 +144,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti .collect(Collectors.toMap(TaskDefinition::getCode, Function.identity())); final TaskInstance taskInstance = taskInstanceDao.queryById(taskExecutionContext.getTaskInstanceId()); - List dependentExecutes = dependentParameters.getDependTaskList() + List dependentExecutes = dependTaskList .stream() .map(dependentTaskModel -> { for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { @@ -210,7 +212,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti } dependResultList.add(dependResult); } - return DependentUtils.getDependResultForRelation(this.dependentParameters.getRelation(), + return DependentUtils.getDependResultForRelation(dependentParameters.getDependence().getRelation(), dependResultList); } @@ -218,7 +220,8 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti boolean isAllDependentTaskFinished = true; for (DependentExecute dependentExecute : dependentTaskList) { if (!dependentExecute.finish(dependentDate, processInstance.getTestFlag(), - dependentParameters.getFailurePolicy(), dependentParameters.getFailureWaitingTime())) { + dependentParameters.getDependence().getFailurePolicy(), + dependentParameters.getDependence().getFailureWaitingTime())) { isAllDependentTaskFinished = false; } dependentExecute.getDependResultMap().forEach((dependentKey, dependResult) -> { @@ -238,7 +241,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti @Override public @NonNull Duration getAsyncTaskStateCheckInterval() { - return dependentParameters.getCheckInterval() == null ? DEFAULT_STATE_CHECK_INTERVAL - : Duration.ofSeconds(dependentParameters.getCheckInterval()); + return dependentParameters.getDependence().getCheckInterval() == null ? DEFAULT_STATE_CHECK_INTERVAL + : Duration.ofSeconds(dependentParameters.getDependence().getCheckInterval()); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java index e6ee3a78ca..774c6cb1f1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.runner.task.dependent; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; @@ -26,8 +27,6 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; -import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; -import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction; @@ -37,6 +36,8 @@ import java.util.Date; import lombok.extern.slf4j.Slf4j; +import com.fasterxml.jackson.core.type.TypeReference; + @Slf4j public class DependentLogicTask extends BaseAsyncLogicTask { @@ -48,7 +49,7 @@ public class DependentLogicTask extends BaseAsyncLogicTask private final TaskInstanceDao taskInstanceDao; private final ProcessInstanceDao processInstanceDao; - private final ProcessInstanceExecCacheManager processInstanceExecCacheManager; + private final WorkflowExecuteRunnable workflowExecuteRunnable; public DependentLogicTask(TaskExecutionContext taskExecutionContext, ProjectDao projectDao, @@ -56,19 +57,16 @@ public class DependentLogicTask extends BaseAsyncLogicTask TaskDefinitionDao taskDefinitionDao, TaskInstanceDao taskInstanceDao, ProcessInstanceDao processInstanceDao, - ProcessInstanceExecCacheManager processInstanceExecCacheManager) throws LogicTaskInitializeException { + WorkflowExecuteRunnable workflowExecuteRunnable) { super(taskExecutionContext, - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) - .getTaskInstance(taskExecutionContext.getTaskInstanceId()) - .orElseThrow(() -> new LogicTaskInitializeException( - "Cannot find the task instance in workflow execute runnable")) - .getDependency()); + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() { + })); this.projectDao = projectDao; this.processDefinitionDao = processDefinitionDao; this.taskDefinitionDao = taskDefinitionDao; this.taskInstanceDao = taskInstanceDao; this.processInstanceDao = processInstanceDao; - this.processInstanceExecCacheManager = processInstanceExecCacheManager; + this.workflowExecuteRunnable = workflowExecuteRunnable; } @@ -85,8 +83,6 @@ public class DependentLogicTask extends BaseAsyncLogicTask @Override public void pause() throws MasterTaskExecuteException { - WorkflowExecuteRunnable workflowExecuteRunnable = - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()); if (workflowExecuteRunnable == null) { log.error("Cannot find the WorkflowExecuteRunnable"); return; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java index e8012ac3f9..970dd964d1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; +import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import lombok.extern.slf4j.Slf4j; @@ -52,6 +53,12 @@ public class DependentLogicTaskPluginFactory implements ILogicTaskPluginFactory< @Override public DependentLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException { + int workflowInstanceId = taskExecutionContext.getProcessInstanceId(); + WorkflowExecuteRunnable workflowExecuteRunnable = + processInstanceExecCacheManager.getByProcessInstanceId(workflowInstanceId); + if (workflowExecuteRunnable == null) { + throw new LogicTaskInitializeException("Cannot find the WorkflowExecuteRunnable: " + workflowInstanceId); + } return new DependentLogicTask( taskExecutionContext, projectDao, @@ -59,7 +66,7 @@ public class DependentLogicTaskPluginFactory implements ILogicTaskPluginFactory< taskDefinitionDao, taskInstanceDao, processInstanceDao, - processInstanceExecCacheManager); + workflowExecuteRunnable); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java index 1883a27d8b..5a44966bca 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventLis import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; -import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction; @@ -41,29 +40,27 @@ import com.fasterxml.jackson.core.type.TypeReference; public class SubWorkflowLogicTask extends BaseAsyncLogicTask { public static final String TASK_TYPE = "SUB_PROCESS"; - private final ProcessInstanceExecCacheManager processInstanceExecCacheManager; + private final WorkflowExecuteRunnable workflowExecuteRunnable; private final ProcessInstanceDao processInstanceDao; public SubWorkflowLogicTask(TaskExecutionContext taskExecutionContext, - ProcessInstanceExecCacheManager processInstanceExecCacheManager, + WorkflowExecuteRunnable workflowExecuteRunnable, ProcessInstanceDao processInstanceDao) { super(taskExecutionContext, JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() { })); - this.processInstanceExecCacheManager = processInstanceExecCacheManager; + this.workflowExecuteRunnable = workflowExecuteRunnable; this.processInstanceDao = processInstanceDao; } @Override - public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() throws MasterTaskExecuteException { + public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() { // todo: create sub workflow instance here? return new SubWorkflowAsyncTaskExecuteFunction(taskExecutionContext, processInstanceDao); } @Override public void pause() throws MasterTaskExecuteException { - WorkflowExecuteRunnable workflowExecuteRunnable = - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()); if (workflowExecuteRunnable == null) { log.warn("Cannot find WorkflowExecuteRunnable"); return; @@ -99,8 +96,6 @@ public class SubWorkflowLogicTask extends BaseAsyncLogicTask { @@ -47,121 +52,96 @@ public class SwitchLogicTask extends BaseSyncLogicTask { private final WorkflowExecuteRunnable workflowExecuteRunnable; private final TaskInstance taskInstance; - public SwitchLogicTask(TaskExecutionContext taskExecutionContext, - ProcessInstanceExecCacheManager processInstanceExecCacheManager) throws LogicTaskInitializeException { - super(taskExecutionContext, - // todo: we need to refactor the logic task parameter........ - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()) - .getTaskInstance(taskExecutionContext.getTaskInstanceId()) - .orElseThrow(() -> new LogicTaskInitializeException( - "Cannot find the task instance in workflow execute runnable")) - .getSwitchDependency()); - this.workflowExecuteRunnable = - processInstanceExecCacheManager.getByProcessInstanceId(taskExecutionContext.getProcessInstanceId()); - this.taskInstance = workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId()) - .orElseThrow(() -> new LogicTaskInitializeException( - "Cannot find the task instance in workflow execute runnable")); - // Since the default branch is not in the dependTaskList, we need to add it to the end - // otherwise the default branch will never be skipped in DAGHelper - addDefaultBranchToEnd(); + public SwitchLogicTask(WorkflowExecuteRunnable workflowExecuteRunnable, + TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException { + super(workflowExecuteRunnable, + taskExecutionContext, + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference() { + })); + this.workflowExecuteRunnable = workflowExecuteRunnable; + this.taskInstance = + workflowExecuteRunnable.getTaskInstance(taskExecutionContext.getTaskInstanceId()).orElseThrow( + () -> new LogicTaskInitializeException( + "Cannot find the task instance in workflow execute runnable")); } @Override public void handle() throws MasterTaskExecuteException { - if (CollectionUtils.isEmpty(taskParameters.getDependTaskList())) { + if (CollectionUtils.isEmpty(taskParameters.getSwitchResult().getDependTaskList())) { // If the branch is empty then will go into the default branch // This case shouldn't happen, we can directly throw exception and forbid the user to set branch + log.info("The switch items is empty"); moveToDefaultBranch(); } else { calculateSwitchBranch(); } - taskInstance.setSwitchDependency(taskParameters); + checkIfBranchExist(taskParameters.getNextBranch()); + taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters)); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS); log.info("Switch task execute finished: {}", taskExecutionContext.getCurrentExecutionStatus().name()); } private void moveToDefaultBranch() { - List switchResultVos = taskParameters.getDependTaskList(); - SwitchResultVo defaultSwitchResultVo = getDefaultSwitchResultVo(); - checkIfBranchExist(defaultSwitchResultVo.getNextNode()); - - taskParameters.setResultConditionLocation(switchResultVos.size() - 1); - + log.info("Begin to move to the default branch"); + if (taskParameters.getSwitchResult().getNextNode() == null) { + throw new IllegalArgumentException( + "The default branch is empty, please check the switch task configuration"); + } + taskParameters.setNextBranch(taskParameters.getSwitchResult().getNextNode()); log.info("The condition is not satisfied, move to the default branch: {}", - taskParameters.getNextNode().stream().map(node -> workflowExecuteRunnable.getWorkflowExecuteContext() - .getWorkflowGraph().getDag().getNode(node).getName()).collect(Collectors.toList())); + getTaskName(taskParameters.getNextBranch())); } private void calculateSwitchBranch() { - List switchResultVos = taskParameters.getDependTaskList(); + List switchResultVos = taskParameters.getSwitchResult().getDependTaskList(); Map globalParams = taskExecutionContext.getPrepareParamsMap(); Map varParams = JSONUtils .toList(taskInstance.getVarPool(), Property.class) .stream() .collect(Collectors.toMap(Property::getProp, Property -> Property)); - int finalConditionLocation = -1; - // The last one is the default branch, no need to calculate - for (int i = 0; i < switchResultVos.size() - 1; i++) { - SwitchResultVo switchResultVo = switchResultVos.get(i); - log.info("Begin to execute {} condition: {} ", i, switchResultVo.getCondition()); - String content = SwitchTaskUtils.generateContentWithTaskParams(switchResultVo.getCondition(), globalParams, - varParams); - log.info("Format condition sentence::{} successfully", content); - boolean result; + Long nextBranch = null; + for (SwitchResultVo switchResultVo : switchResultVos) { + log.info("Begin to execute switch item: {} ", switchResultVo); try { - result = SwitchTaskUtils.evaluate(content); - log.info("Execute condition sentence: {} successfully: {}", content, result); - if (result) { + String content = SwitchTaskUtils.generateContentWithTaskParams(switchResultVo.getCondition(), + globalParams, varParams); + log.info("Format condition sentence::{} successfully", content); + boolean conditionResult = SwitchTaskUtils.evaluate(content); + log.info("Execute condition sentence: {} successfully: {}", content, conditionResult); + if (conditionResult) { // If matched, break the loop - finalConditionLocation = i; + nextBranch = switchResultVo.getNextNode(); break; } } catch (Exception e) { - log.info("Execute condition sentence: {} failed", content, e); + log.info("Execute switch item: {} failed", switchResultVo, e); } } - // If the finalConditionLocation is -1, then the default branch will be executed - if (finalConditionLocation >= 0) { - List nextNodes = switchResultVos.get(finalConditionLocation).getNextNode(); - checkIfBranchExist(nextNodes); - log.info("The condition is satisfied, move to the branch: {}", - switchResultVos.get(finalConditionLocation).getNextNode().stream() - .map(node -> workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowGraph().getDag() - .getNode(node).getName()) - .collect(Collectors.toList())); - taskParameters.setResultConditionLocation(finalConditionLocation); - } else { + + if (nextBranch == null) { + log.info("All switch item is not satisfied"); moveToDefaultBranch(); } } - private void checkIfBranchExist(List branchNode) { - if (CollectionUtils.isEmpty(branchNode)) { - throw new IllegalArgumentException("The branchNode is empty, please check the switch task configuration"); + private void checkIfBranchExist(Long branchNode) { + if (branchNode == null) { + throw new IllegalArgumentException("The branch is empty, please check the switch task configuration"); } - for (Long branch : branchNode) { - if (branch == null) { - throw new IllegalArgumentException("The branch is empty, please check the switch task configuration"); - } - if (!workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowGraph().getDag().containsNode(branch)) { - throw new IllegalArgumentException( - "The branch(code= " + branchNode - + ") is not in the dag, please check the switch task configuration"); - } - } - } - - private void addDefaultBranchToEnd() { - SwitchResultVo switchResultVo = new SwitchResultVo(null, taskParameters.getNextNode()); - List dependTaskList = taskParameters.getDependTaskList(); - if (!dependTaskList.contains(switchResultVo)) { - dependTaskList.add(switchResultVo); + if (!workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowGraph().getDag().containsNode(branchNode)) { + throw new IllegalArgumentException( + "The branch(code= " + branchNode + + ") is not in the dag, please check the switch task configuration"); } } - private SwitchResultVo getDefaultSwitchResultVo() { - return taskParameters.getDependTaskList().get(taskParameters.getDependTaskList().size() - 1); + private String getTaskName(Long taskCode) { + Optional taskNode = Optional.ofNullable(workflowExecuteRunnable.getWorkflowExecuteContext()) + .map(IWorkflowExecuteContext::getWorkflowGraph) + .map(IWorkflowGraph::getDag) + .map(dag -> dag.getNode(taskCode)); + return taskNode.map(TaskNode::getName).orElse(null); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java index dfb47aa92d..926e70c637 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.runner.task.switchtask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; +import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import lombok.extern.slf4j.Slf4j; @@ -36,7 +37,14 @@ public class SwitchLogicTaskPluginFactory implements ILogicTaskPluginFactory MASTER_TASK_TYPES = Sets.newHashSet( - BlockingLogicTask.TASK_TYPE, - ConditionLogicTask.TASK_TYPE, - DependentLogicTask.TASK_TYPE, - SubWorkflowLogicTask.TASK_TYPE, - SwitchLogicTask.TASK_TYPE, - DynamicLogicTask.TASK_TYPE); - - public boolean isMasterTask(String taskType) { - return MASTER_TASK_TYPES.contains(taskType); - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java index 409f1b7691..0adc938d5c 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java @@ -392,7 +392,6 @@ public class WorkflowExecuteRunnableTest { // task instance already finished, not dispatch TaskInstance taskInstance = new TaskInstance(); taskInstance.setState(TaskExecutionStatus.PAUSE); - Mockito.when(processInstance.isBlocked()).thenReturn(true); TaskExecuteRunnable taskExecuteRunnable = Mockito.mock(TaskExecuteRunnable.class); workflowExecuteThread.tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable); Mockito.verify(taskExecuteRunnable, Mockito.never()).dispatch(); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactoryTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactoryTest.java index 93c8465362..dd63abcfa9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactoryTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactoryTest.java @@ -17,15 +17,18 @@ package org.apache.dolphinscheduler.server.master.runner.dispatcher; -import org.apache.dolphinscheduler.server.master.runner.task.blocking.BlockingLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.condition.ConditionLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.dependent.DependentLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.subworkflow.SubWorkflowLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.switchtask.SwitchLogicTask; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.DynamicLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory; + import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -42,18 +45,20 @@ public class TaskDispatchFactoryTest { @Mock private WorkerTaskDispatcher workerTaskDispatcher; - @Test - public void getTaskDispatcher() { - Assertions.assertEquals(masterTaskDispatcher, - taskDispatchFactory.getTaskDispatcher(BlockingLogicTask.TASK_TYPE)); - Assertions.assertEquals(masterTaskDispatcher, - taskDispatchFactory.getTaskDispatcher(ConditionLogicTask.TASK_TYPE)); - Assertions.assertEquals(masterTaskDispatcher, - taskDispatchFactory.getTaskDispatcher(DependentLogicTask.TASK_TYPE)); - Assertions.assertEquals(masterTaskDispatcher, - taskDispatchFactory.getTaskDispatcher(SubWorkflowLogicTask.TASK_TYPE)); - Assertions.assertEquals(masterTaskDispatcher, taskDispatchFactory.getTaskDispatcher(SwitchLogicTask.TASK_TYPE)); - - Assertions.assertEquals(workerTaskDispatcher, taskDispatchFactory.getTaskDispatcher("SHELL")); + @ParameterizedTest + @ValueSource(strings = { + ConditionsLogicTaskChannelFactory.NAME, + DependentLogicTaskChannelFactory.NAME, + DynamicLogicTaskChannelFactory.NAME, + SubWorkflowLogicTaskChannelFactory.NAME, + SwitchLogicTaskChannelFactory.NAME}) + public void getTaskDispatcher_withLogicTask(String taskType) { + assertThat(taskDispatchFactory.getTaskDispatcher(taskType)).isSameInstanceAs(masterTaskDispatcher); + } + + @ParameterizedTest + @ValueSource(strings = {ShellTaskChannelFactory.NAME}) + public void getTaskDispatcher_withWorkerTask(String taskType) { + assertThat(taskDispatchFactory.getTaskDispatcher(taskType)).isSameInstanceAs(workerTaskDispatcher); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java index a5b9f5a371..019b435fe0 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.master.runner.task.dynamic; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; @@ -43,8 +44,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import com.fasterxml.jackson.databind.ObjectMapper; - @ExtendWith(MockitoExtension.class) class DynamicLogicTaskTest { @@ -74,14 +73,11 @@ class DynamicLogicTaskTest { private DynamicLogicTask dynamicLogicTask; - private ObjectMapper objectMapper; - @BeforeEach public void setUp() { // Set up your test environment before each test. dynamicParameters = new DynamicParameters(); taskExecutionContext = Mockito.mock(TaskExecutionContext.class); - objectMapper = new ObjectMapper(); processInstance = new ProcessInstance(); Mockito.when(processInstanceDao.queryById(Mockito.any())).thenReturn(processInstance); dynamicLogicTask = new DynamicLogicTask( @@ -95,7 +91,7 @@ class DynamicLogicTaskTest { } @Test - void testGenerateParameterGroup() throws Exception { + void testGenerateParameterGroup() { DynamicInputParameter dynamicInputParameter1 = new DynamicInputParameter(); dynamicInputParameter1.setName("param1"); dynamicInputParameter1.setValue("a,b,c"); @@ -113,7 +109,7 @@ class DynamicLogicTaskTest { Mockito.when(taskExecutionContext.getPrepareParamsMap()).thenReturn(new HashMap<>()); Mockito.when(taskExecutionContext.getTaskParams()) - .thenReturn(objectMapper.writeValueAsString(dynamicParameters)); + .thenReturn(JSONUtils.toJsonString(dynamicParameters)); dynamicLogicTask = new DynamicLogicTask( taskExecutionContext, diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java index 2081d2dd7a..7e6f30970f 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.service; -import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_TASK_TYPE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.doNothing; @@ -31,6 +29,8 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.shell.ShellTaskChannelFactory; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; @@ -139,13 +139,13 @@ public class FailoverServiceTest { masterTaskInstance.setId(1); masterTaskInstance.setStartTime(new Date()); masterTaskInstance.setHost(testMasterHost); - masterTaskInstance.setTaskType(TASK_TYPE_SWITCH); + masterTaskInstance.setTaskType(SwitchLogicTaskChannelFactory.NAME); workerTaskInstance = new TaskInstance(); workerTaskInstance.setId(2); workerTaskInstance.setStartTime(new Date()); workerTaskInstance.setHost(testWorkerHost); - workerTaskInstance.setTaskType(COMMON_TASK_TYPE); + workerTaskInstance.setTaskType(ShellTaskChannelFactory.NAME); given(processService.queryNeedFailoverProcessInstances(Mockito.anyString())) .willReturn(Arrays.asList(processInstance)); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/TaskUtilsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/TaskUtilsTest.java deleted file mode 100644 index bec04b1936..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/TaskUtilsTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.utils; - -import org.apache.dolphinscheduler.server.master.runner.task.blocking.BlockingLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.condition.ConditionLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.dependent.DependentLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.subworkflow.SubWorkflowLogicTask; -import org.apache.dolphinscheduler.server.master.runner.task.switchtask.SwitchLogicTask; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TaskUtilsTest { - - @Test - public void isMasterTask() { - Assertions.assertTrue(TaskUtils.isMasterTask(BlockingLogicTask.TASK_TYPE)); - Assertions.assertTrue(TaskUtils.isMasterTask(ConditionLogicTask.TASK_TYPE)); - Assertions.assertTrue(TaskUtils.isMasterTask(DependentLogicTask.TASK_TYPE)); - Assertions.assertTrue(TaskUtils.isMasterTask(SubWorkflowLogicTask.TASK_TYPE)); - Assertions.assertTrue(TaskUtils.isMasterTask(SwitchLogicTask.TASK_TYPE)); - } -} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java index 954003ec41..c9601f15b5 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java @@ -17,27 +17,17 @@ package org.apache.dolphinscheduler.service.model; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH; - import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.common.model.PreviousTaskNode; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; -import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -118,11 +108,6 @@ public class TaskNode { @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) private String preTasks; - /** - * node dependency list - */ - private List preTaskNodeList; - /** * users store additional information */ @@ -135,25 +120,6 @@ public class TaskNode { */ private List depList; - /** - * outer dependency information - */ - @JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) - @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) - private String dependence; - - @JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) - @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) - private String conditionResult; - - @JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) - @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) - private String switchResult; - - @JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) - @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) - private String waitStartTimeout; - /** * task instance priority */ @@ -312,10 +278,8 @@ public class TaskNode { && Objects.equals(preTasks, taskNode.preTasks) && Objects.equals(extras, taskNode.extras) && Objects.equals(runFlag, taskNode.runFlag) - && Objects.equals(dependence, taskNode.dependence) && Objects.equals(workerGroup, taskNode.workerGroup) && Objects.equals(environmentCode, taskNode.environmentCode) - && Objects.equals(conditionResult, taskNode.conditionResult) && CollectionUtils.isEqualCollection(depList, taskNode.depList) && Objects.equals(taskExecuteType, taskNode.taskExecuteType); } @@ -325,14 +289,6 @@ public class TaskNode { return Objects.hash(name, desc, type, params, preTasks, extras, depList, runFlag); } - public String getDependence() { - return dependence; - } - - public void setDependence(String dependence) { - this.dependence = dependence; - } - public int getMaxRetryTimes() { return maxRetryTimes; } @@ -373,14 +329,6 @@ public class TaskNode { this.workerGroup = workerGroup; } - public String getConditionResult() { - return conditionResult; - } - - public void setConditionResult(String conditionResult) { - this.conditionResult = conditionResult; - } - public int getDelayTime() { return delayTime; } @@ -405,64 +353,6 @@ public class TaskNode { this.version = version; } - /** - * get task time out parameter - * - * @return task time out parameter - */ - public TaskTimeoutParameter getTaskTimeoutParameter() { - if (!StringUtils.isEmpty(this.getTimeout())) { - String formatStr = - String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); - String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name()); - return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class); - } - return new TaskTimeoutParameter(false); - } - - public boolean isConditionsTask() { - return TASK_TYPE_CONDITIONS.equalsIgnoreCase(this.getType()); - } - - public boolean isSwitchTask() { - return TASK_TYPE_SWITCH.equalsIgnoreCase(this.getType()); - } - - public List getPreTaskNodeList() { - return preTaskNodeList; - } - - public boolean isBlockingTask() { - return TASK_TYPE_BLOCKING.equalsIgnoreCase(this.getType()); - } - - public void setPreTaskNodeList(List preTaskNodeList) { - this.preTaskNodeList = preTaskNodeList; - } - - public String getTaskParams() { - Map taskParams = JSONUtils.parseObject(this.params, new TypeReference>() { - }); - - if (taskParams == null) { - taskParams = new HashMap<>(); - } - taskParams.put(Constants.CONDITION_RESULT, this.conditionResult); - taskParams.put(Constants.DEPENDENCE, this.dependence); - taskParams.put(Constants.SWITCH_RESULT, this.switchResult); - taskParams.put(Constants.WAIT_START_TIMEOUT, this.waitStartTimeout); - return JSONUtils.toJsonString(taskParams); - } - - public Map taskParamsToJsonObj(String taskParams) { - Map taskParamsMap = JSONUtils.parseObject(taskParams, new TypeReference>() { - }); - if (taskParamsMap == null) { - taskParamsMap = new HashMap<>(); - } - return taskParamsMap; - } - @Override public String toString() { return "TaskNode{" @@ -478,11 +368,8 @@ public class TaskNode { + ", retryInterval=" + retryInterval + ", params='" + params + '\'' + ", preTasks='" + preTasks + '\'' - + ", preTaskNodeList=" + preTaskNodeList + ", extras='" + extras + '\'' + ", depList=" + depList - + ", dependence='" + dependence + '\'' - + ", conditionResult='" + conditionResult + '\'' + ", taskInstancePriority=" + taskInstancePriority + ", workerGroup='" + workerGroup + '\'' + ", environmentCode=" + environmentCode @@ -500,22 +387,6 @@ public class TaskNode { return this.environmentCode; } - public String getSwitchResult() { - return switchResult; - } - - public void setSwitchResult(String switchResult) { - this.switchResult = switchResult; - } - - public String getWaitStartTimeout() { - return this.waitStartTimeout; - } - - public void setWaitStartTimeout(String waitStartTimeout) { - this.waitStartTimeout = waitStartTimeout; - } - public int getTaskGroupId() { return taskGroupId; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 8787aabc8b..9a5692b4c0 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -176,8 +176,6 @@ public interface ProcessService { DqComparisonType getComparisonTypeById(int id); - void changeTaskGroupQueueStatus(int taskId, TaskGroupQueueStatus status); - TaskGroupQueue insertIntoTaskGroupQueue(Integer taskId, String taskName, Integer groupId, diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 470226bec0..cfc1fc3a8c 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -120,6 +120,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; @@ -1051,7 +1052,7 @@ public class ProcessServiceImpl implements ProcessService { */ private void initTaskInstance(TaskInstance taskInstance) { - if (!taskInstance.isSubProcess() + if (!TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType()) && (taskInstance.getState().isKill() || taskInstance.getState().isFailure())) { taskInstance.setFlag(Flag.NO); taskInstanceDao.updateById(taskInstance); @@ -1169,7 +1170,7 @@ public class ProcessServiceImpl implements ProcessService { */ @Override public void createSubWorkProcess(ProcessInstance parentProcessInstance, TaskInstance task) { - if (!task.isSubProcess()) { + if (!TaskTypeUtils.isSubWorkflowTask(task.getTaskType())) { return; } // check create sub work flow firstly @@ -1940,13 +1941,7 @@ public class ProcessServiceImpl implements ProcessService { : Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); taskNode.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes()); taskNode.setRetryInterval(taskDefinitionLog.getFailRetryInterval()); - Map taskParamsMap = taskNode.taskParamsToJsonObj(taskDefinitionLog.getTaskParams()); - taskNode.setConditionResult(JSONUtils.toJsonString(taskParamsMap.get(Constants.CONDITION_RESULT))); - taskNode.setSwitchResult(JSONUtils.toJsonString(taskParamsMap.get(Constants.SWITCH_RESULT))); - taskNode.setDependence(JSONUtils.toJsonString(taskParamsMap.get(Constants.DEPENDENCE))); - taskParamsMap.remove(Constants.CONDITION_RESULT); - taskParamsMap.remove(Constants.DEPENDENCE); - taskNode.setParams(JSONUtils.toJsonString(taskParamsMap)); + taskNode.setParams(taskDefinitionLog.getTaskParams()); taskNode.setTaskInstancePriority(taskDefinitionLog.getTaskPriority()); taskNode.setWorkerGroup(taskDefinitionLog.getWorkerGroup()); taskNode.setEnvironmentCode(taskDefinitionLog.getEnvironmentCode()); @@ -2039,22 +2034,6 @@ public class ProcessServiceImpl implements ProcessService { return dqComparisonTypeMapper.selectById(id); } - /** - * release the TGQ resource when the corresponding task is finished. - * - * @param taskId task id - * @return the result code and msg - */ - - @Override - public void changeTaskGroupQueueStatus(int taskId, TaskGroupQueueStatus status) { - TaskGroupQueue taskGroupQueue = taskGroupQueueMapper.queryByTaskId(taskId); - taskGroupQueue.setInQueue(Flag.NO.getCode()); - taskGroupQueue.setStatus(status); - taskGroupQueue.setUpdateTime(new Date(System.currentTimeMillis())); - taskGroupQueueMapper.updateById(taskGroupQueue); - } - @Override public TaskGroupQueue insertIntoTaskGroupQueue(Integer taskInstanceId, String taskName, diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java index 4a6235c0e0..7e6cbda608 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java @@ -23,10 +23,12 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; +import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessDag; @@ -35,6 +37,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -44,6 +47,9 @@ import java.util.Set; import lombok.extern.slf4j.Slf4j; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Lists; + /** * dag tools */ @@ -228,22 +234,6 @@ public class DagHelper { return processDag; } - /** - * find node by node name - * - * @param nodeDetails nodeDetails - * @param nodeName nodeName - * @return task node - */ - public static TaskNode findNodeByName(List nodeDetails, String nodeName) { - for (TaskNode taskNode : nodeDetails) { - if (taskNode.getName().equals(nodeName)) { - return taskNode; - } - } - return null; - } - /** * find node by node code * @@ -305,10 +295,10 @@ public class DagHelper { if (preNodeCode == null) { startVertexes = dag.getBeginNode(); - } else if (dag.getNode(preNodeCode).isConditionsTask()) { + } else if (TaskTypeUtils.isConditionTask(dag.getNode(preNodeCode).getType())) { List conditionTaskList = parseConditionTask(preNodeCode, skipTaskNodeList, dag, completeTaskList); startVertexes.addAll(conditionTaskList); - } else if (dag.getNode(preNodeCode).isSwitchTask()) { + } else if (TaskTypeUtils.isSwitchTask(dag.getNode(preNodeCode).getType())) { List conditionTaskList = parseSwitchTask(preNodeCode, skipTaskNodeList, dag, completeTaskList); startVertexes.addAll(conditionTaskList); } else { @@ -321,7 +311,7 @@ public class DagHelper { continue; } if (isTaskNodeNeedSkip(taskNode, skipTaskNodeList)) { - setTaskNodeSkip(subsequent, dag, completeTaskList, skipTaskNodeList); + setTaskNodeSkip(subsequent, dag, skipTaskNodeList); continue; } if (!DagHelper.allDependsForbiddenOrEnd(taskNode, dag, skipTaskNodeList, completeTaskList)) { @@ -362,18 +352,20 @@ public class DagHelper { Map completeTaskList) { List conditionTaskList = new ArrayList<>(); TaskNode taskNode = dag.getNode(nodeCode); - if (!taskNode.isConditionsTask()) { + if (!TaskTypeUtils.isConditionTask(taskNode.getType())) { return conditionTaskList; } if (!completeTaskList.containsKey(nodeCode)) { return conditionTaskList; } TaskInstance taskInstance = completeTaskList.get(nodeCode); - ConditionsParameters conditionsParameters = taskInstance.getConditionsParameters(); - ConditionsParameters.ConditionResult conditionResult = taskInstance.getConditionResult(); + ConditionsParameters conditionsParameters = + JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference() { + }); + ConditionsParameters.ConditionResult conditionResult = conditionsParameters.getConditionResult(); List skipNodeList = new ArrayList<>(); - if (conditionsParameters.isConditionSuccess()) { + if (conditionResult.isConditionSuccess()) { conditionTaskList = conditionResult.getSuccessNode(); skipNodeList = conditionResult.getFailedNode(); } else { @@ -382,7 +374,7 @@ public class DagHelper { } if (CollectionUtils.isNotEmpty(skipNodeList)) { - skipNodeList.forEach(skipNode -> setTaskNodeSkip(skipNode, dag, completeTaskList, skipTaskNodeList)); + skipNodeList.forEach(skipNode -> setTaskNodeSkip(skipNode, dag, skipTaskNodeList)); } // the conditionTaskList maybe null if no next task conditionTaskList = Optional.ofNullable(conditionTaskList).orElse(new ArrayList<>()); @@ -402,82 +394,50 @@ public class DagHelper { Map completeTaskList) { List conditionTaskList = new ArrayList<>(); TaskNode taskNode = dag.getNode(nodeCode); - if (!taskNode.isSwitchTask()) { + if (!SwitchLogicTaskChannelFactory.NAME.equals(taskNode.getType())) { return conditionTaskList; } if (!completeTaskList.containsKey(nodeCode)) { return conditionTaskList; } - conditionTaskList = skipTaskNode4Switch(taskNode, skipTaskNodeList, completeTaskList, dag); + conditionTaskList = skipTaskNode4Switch(skipTaskNodeList, completeTaskList.get(nodeCode), dag); return conditionTaskList; } - public static List skipTaskNode4Switch(TaskNode taskNode, - Map skipTaskNodeList, - Map completeTaskList, + public static List skipTaskNode4Switch(Map skipTaskNodeList, + TaskInstance taskInstance, DAG dag) { - SwitchParameters switchParameters = - completeTaskList.get(taskNode.getCode()).getSwitchDependency(); - int resultConditionLocation = switchParameters.getResultConditionLocation(); - List conditionResultVoList = switchParameters.getDependTaskList(); - List switchTaskList = conditionResultVoList.get(resultConditionLocation).getNextNode(); - Set switchNeedWorkCodes = new HashSet<>(); - if (CollectionUtils.isEmpty(switchTaskList)) { - return new ArrayList<>(); - } - // get all downstream nodes of the branch that the switch node needs to execute - for (Long switchTaskCode : switchTaskList) { - getSwitchNeedWorkCodes(switchTaskCode, dag, switchNeedWorkCodes); - } - // conditionResultVoList.remove(resultConditionLocation); - for (SwitchResultVo info : conditionResultVoList) { - if (CollectionUtils.isEmpty(info.getNextNode())) { - continue; - } - for (Long nextNode : info.getNextNode()) { - setSwitchTaskNodeSkip(nextNode, dag, completeTaskList, skipTaskNodeList, - switchNeedWorkCodes); - } - } - return switchTaskList; - } + JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference() { + }); - /** - * get all downstream nodes of the branch that the switch node needs to execute - * - * @param taskCode - * @param dag - * @param switchNeedWorkCodes - */ - public static void getSwitchNeedWorkCodes(Long taskCode, DAG dag, - Set switchNeedWorkCodes) { - switchNeedWorkCodes.add(taskCode); - Set subsequentNodes = dag.getSubsequentNodes(taskCode); - if (org.apache.commons.collections.CollectionUtils.isNotEmpty(subsequentNodes)) { - for (Long subCode : subsequentNodes) { - getSwitchNeedWorkCodes(subCode, dag, switchNeedWorkCodes); - } + SwitchParameters.SwitchResult switchResult = switchParameters.getSwitchResult(); + Long nextBranch = switchParameters.getNextBranch(); + if (switchResult == null) { + log.error("switchResult is null, please check the switch task configuration"); + return Collections.emptyList(); + } + if (nextBranch == null) { + log.error("switchParameters.getNextBranch() is null, please check the switch task configuration"); + return Collections.emptyList(); } - } - private static void setSwitchTaskNodeSkip(Long skipNodeCode, - DAG dag, - Map completeTaskList, - Map skipTaskNodeList, - Set switchNeedWorkCodes) { - // ignore when the node that needs to be skipped exists on the branch that the switch type node needs to execute - if (!dag.containsNode(skipNodeCode) || switchNeedWorkCodes.contains(skipNodeCode)) { - return; + Set allNextBranches = new HashSet<>(); + if (switchResult.getNextNode() != null) { + allNextBranches.add(switchResult.getNextNode()); } - skipTaskNodeList.putIfAbsent(skipNodeCode, dag.getNode(skipNodeCode)); - Collection postNodeList = dag.getSubsequentNodes(skipNodeCode); - for (Long post : postNodeList) { - TaskNode postNode = dag.getNode(post); - if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) { - setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList); + if (CollectionUtils.isNotEmpty(switchResult.getDependTaskList())) { + for (SwitchResultVo switchResultVo : switchResult.getDependTaskList()) { + allNextBranches.add(switchResultVo.getNextNode()); } } + + allNextBranches.remove(nextBranch); + + for (Long branch : allNextBranches) { + setTaskNodeSkip(branch, dag, skipTaskNodeList); + } + return Lists.newArrayList(nextBranch); } /** @@ -485,7 +445,6 @@ public class DagHelper { */ private static void setTaskNodeSkip(Long skipNodeCode, DAG dag, - Map completeTaskList, Map skipTaskNodeList) { if (!dag.containsNode(skipNodeCode)) { return; @@ -495,7 +454,7 @@ public class DagHelper { for (Long post : postNodeList) { TaskNode postNode = dag.getNode(post); if (isTaskNodeNeedSkip(postNode, skipTaskNodeList)) { - setTaskNodeSkip(post, dag, completeTaskList, skipTaskNodeList); + setTaskNodeSkip(post, dag, skipTaskNodeList); } } } @@ -591,31 +550,7 @@ public class DagHelper { */ public static boolean haveConditionsAfterNode(Long parentNodeCode, DAG dag) { - return haveSubAfterNode(parentNodeCode, dag, TaskConstants.TASK_TYPE_CONDITIONS); - } - - /** - * is there have conditions after the parent node - */ - public static boolean haveConditionsAfterNode(Long parentNodeCode, List taskNodes) { - if (CollectionUtils.isEmpty(taskNodes)) { - return false; - } - for (TaskNode taskNode : taskNodes) { - List preTasksList = JSONUtils.toList(taskNode.getPreTasks(), Long.class); - if (preTasksList.contains(parentNodeCode) && taskNode.isConditionsTask()) { - return true; - } - } - return false; - } - - /** - * is there have blocking node after the parent node - */ - public static boolean haveBlockingAfterNode(Long parentNodeCode, - DAG dag) { - return haveSubAfterNode(parentNodeCode, dag, TaskConstants.TASK_TYPE_BLOCKING); + return haveSubAfterNode(parentNodeCode, dag, ConditionsLogicTaskChannelFactory.NAME); } /** diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java index a36c414272..daaa6ac81d 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java @@ -17,19 +17,19 @@ package org.apache.dolphinscheduler.service.utils; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; - import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; +import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessDag; @@ -44,10 +44,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.Lists; +import com.google.common.truth.Truth; -/** - * dag helper test - */ public class DagHelperTest { @Test @@ -90,26 +89,105 @@ public class DagHelperTest { boolean canSubmit = DagHelper.haveAllNodeAfterNode(parentNodeCode, dag); Assertions.assertTrue(canSubmit); - boolean haveBlocking = DagHelper.haveBlockingAfterNode(parentNodeCode, dag); - Assertions.assertTrue(haveBlocking); - boolean haveConditions = DagHelper.haveConditionsAfterNode(parentNodeCode, dag); Assertions.assertTrue(haveConditions); - boolean dependent = DagHelper.haveSubAfterNode(parentNodeCode, dag, TaskConstants.TASK_TYPE_DEPENDENT); + boolean dependent = DagHelper.haveSubAfterNode(parentNodeCode, dag, DependentLogicTaskChannelFactory.NAME); Assertions.assertFalse(dependent); } - /** - * test task node can submit - * - * @throws JsonProcessingException if error throws JsonProcessingException - */ @Test - public void testTaskNodeCanSubmit() throws IOException { + public void testTaskNodeCanSubmit() { + List taskNodeList = new ArrayList<>(); + TaskNode node1 = new TaskNode(); + node1.setId("1"); + node1.setName("1"); + node1.setCode(1); + node1.setType("SHELL"); + taskNodeList.add(node1); + + TaskNode node2 = new TaskNode(); + node2.setId("2"); + node2.setName("2"); + node2.setCode(2); + node2.setType("SHELL"); + List dep2 = new ArrayList<>(); + dep2.add("1"); + node2.setPreTasks(JSONUtils.toJsonString(dep2)); + taskNodeList.add(node2); + + TaskNode node4 = new TaskNode(); + node4.setId("4"); + node4.setName("4"); + node4.setCode(4); + node4.setType("SHELL"); + taskNodeList.add(node4); + + TaskNode node3 = new TaskNode(); + node3.setId("3"); + node3.setName("3"); + node3.setCode(3); + node3.setType("SHELL"); + List dep3 = new ArrayList<>(); + dep3.add("2"); + dep3.add("4"); + node3.setPreTasks(JSONUtils.toJsonString(dep3)); + taskNodeList.add(node3); + + TaskNode node5 = new TaskNode(); + node5.setId("5"); + node5.setName("5"); + node5.setCode(5); + node5.setType("SHELL"); + List dep5 = new ArrayList<>(); + dep5.add("3"); + dep5.add("8"); + node5.setPreTasks(JSONUtils.toJsonString(dep5)); + taskNodeList.add(node5); + + TaskNode node6 = new TaskNode(); + node6.setId("6"); + node6.setName("6"); + node6.setCode(6); + node6.setType("SHELL"); + List dep6 = new ArrayList<>(); + dep6.add("3"); + node6.setPreTasks(JSONUtils.toJsonString(dep6)); + taskNodeList.add(node6); + + TaskNode node7 = new TaskNode(); + node7.setId("7"); + node7.setName("7"); + node7.setCode(7); + node7.setType("SHELL"); + List dep7 = new ArrayList<>(); + dep7.add("5"); + node7.setPreTasks(JSONUtils.toJsonString(dep7)); + taskNodeList.add(node7); + + TaskNode node8 = new TaskNode(); + node8.setId("8"); + node8.setName("8"); + node8.setCode(8); + node8.setType("SHELL"); + List dep8 = new ArrayList<>(); + dep8.add("2"); + node8.setPreTasks(JSONUtils.toJsonString(dep8)); + taskNodeList.add(node8); + + List startNodes = new ArrayList<>(); + List recoveryNodes = new ArrayList<>(); + List destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList, + startNodes, recoveryNodes, TaskDependType.TASK_POST); + List taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList); + ProcessDag processDag = new ProcessDag(); + processDag.setEdges(taskNodeRelations); + processDag.setNodes(destTaskNodeList); + // 1->2->3->5->7 // 4->3->6 - DAG dag = generateDag(); + // 1->2->8->5->7 + DAG dag = DagHelper.buildDagGraph(processDag); TaskNode taskNode3 = dag.getNode(3L); Map completeTaskList = new HashMap<>(); Map skipNodeList = new HashMap<>(); @@ -117,7 +195,7 @@ public class DagHelperTest { Boolean canSubmit = false; // 2/4 are forbidden submit 3 - TaskNode node2 = dag.getNode(2L); + node2 = dag.getNode(2L); node2.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); TaskNode nodex = dag.getNode(4L); nodex.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); @@ -132,21 +210,107 @@ public class DagHelperTest { Assertions.assertEquals(canSubmit, false); // 2/3 forbidden submit 5 - TaskNode node3 = dag.getNode(3L); + node3 = dag.getNode(3L); node3.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); - TaskNode node8 = dag.getNode(8L); + node8 = dag.getNode(8L); node8.setRunFlag(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); - TaskNode node5 = dag.getNode(5L); + node5 = dag.getNode(5L); canSubmit = DagHelper.allDependsForbiddenOrEnd(node5, dag, skipNodeList, completeTaskList); Assertions.assertEquals(canSubmit, true); } - /** - * test parse post node list - */ @Test - public void testParsePostNodeList() throws IOException { - DAG dag = generateDag(); + public void testParsePostNodeList() { + List taskNodeList = new ArrayList<>(); + TaskNode node1 = new TaskNode(); + node1.setId("1"); + node1.setName("1"); + node1.setCode(1); + node1.setType("SHELL"); + taskNodeList.add(node1); + + TaskNode node2 = new TaskNode(); + node2.setId("2"); + node2.setName("2"); + node2.setCode(2); + node2.setType("SHELL"); + List dep2 = new ArrayList<>(); + dep2.add("1"); + node2.setPreTasks(JSONUtils.toJsonString(dep2)); + taskNodeList.add(node2); + + TaskNode node4 = new TaskNode(); + node4.setId("4"); + node4.setName("4"); + node4.setCode(4); + node4.setType("SHELL"); + taskNodeList.add(node4); + + TaskNode node3 = new TaskNode(); + node3.setId("3"); + node3.setName("3"); + node3.setCode(3); + node3.setType("SHELL"); + List dep3 = new ArrayList<>(); + dep3.add("2"); + dep3.add("4"); + node3.setPreTasks(JSONUtils.toJsonString(dep3)); + taskNodeList.add(node3); + + TaskNode node5 = new TaskNode(); + node5.setId("5"); + node5.setName("5"); + node5.setCode(5); + node5.setType("SHELL"); + List dep5 = new ArrayList<>(); + dep5.add("3"); + dep5.add("8"); + node5.setPreTasks(JSONUtils.toJsonString(dep5)); + taskNodeList.add(node5); + + TaskNode node6 = new TaskNode(); + node6.setId("6"); + node6.setName("6"); + node6.setCode(6); + node6.setType("SHELL"); + List dep6 = new ArrayList<>(); + dep6.add("3"); + node6.setPreTasks(JSONUtils.toJsonString(dep6)); + taskNodeList.add(node6); + + TaskNode node7 = new TaskNode(); + node7.setId("7"); + node7.setName("7"); + node7.setCode(7); + node7.setType("SHELL"); + List dep7 = new ArrayList<>(); + dep7.add("5"); + node7.setPreTasks(JSONUtils.toJsonString(dep7)); + taskNodeList.add(node7); + + TaskNode node8 = new TaskNode(); + node8.setId("8"); + node8.setName("8"); + node8.setCode(8); + node8.setType("SHELL"); + List dep8 = new ArrayList<>(); + dep8.add("2"); + node8.setPreTasks(JSONUtils.toJsonString(dep8)); + taskNodeList.add(node8); + + List startNodes = new ArrayList<>(); + List recoveryNodes = new ArrayList<>(); + List destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList, + startNodes, recoveryNodes, TaskDependType.TASK_POST); + List taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList); + ProcessDag processDag = new ProcessDag(); + processDag.setEdges(taskNodeRelations); + processDag.setNodes(destTaskNodeList); + + // 1->2->3->5->7 + // 4->3->6 + // 1->2->8->5->7 + DAG dag = DagHelper.buildDagGraph(processDag); Map completeTaskList = new HashMap<>(); Map skipNodeList = new HashMap<>(); @@ -206,11 +370,6 @@ public class DagHelperTest { Assertions.assertTrue(postNodes.contains(7L)); } - /** - * test forbidden post node - * - * @throws JsonProcessingException - */ @Test public void testForbiddenPostNode() throws IOException { DAG dag = generateDag(); @@ -244,11 +403,6 @@ public class DagHelperTest { Assertions.assertTrue(postNodes.contains(3L)); } - /** - * test condition post node - * - * @throws JsonProcessingException - */ @Test public void testConditionPostNode() throws IOException { DAG dag = generateDag(); @@ -262,26 +416,18 @@ public class DagHelperTest { completeTaskList.put(4L, new TaskInstance()); TaskInstance taskInstance3 = new TaskInstance(); - taskInstance3.setTaskType(TASK_TYPE_CONDITIONS); - Map params = new HashMap<>(); + taskInstance3.setTaskType(ConditionsLogicTaskChannelFactory.NAME); + ConditionsParameters.ConditionResult conditionResult = ConditionsParameters.ConditionResult.builder() + .conditionSuccess(true) + .successNode(Lists.newArrayList(5L)) + .failedNode(Lists.newArrayList(6L)) + .build(); ConditionsParameters conditionsParameters = new ConditionsParameters(); - conditionsParameters.setConditionSuccess(true); - params.put(Constants.DEPENDENCE, "{\"conditionSuccess\": true}"); - params.put(Constants.CONDITION_RESULT, "{\n" - + - " \"successNode\": [5\n" - + - " ],\n" - + - " \"failedNode\": [6\n" - + - " ]\n" - + - " }"); - taskInstance3.setTaskParams(JSONUtils.toJsonString(params)); + conditionsParameters.setConditionResult(conditionResult); + taskInstance3.setTaskParams(JSONUtils.toJsonString(conditionsParameters)); taskInstance3.setState(TaskExecutionStatus.SUCCESS); TaskNode node3 = dag.getNode(3L); - node3.setType(TASK_TYPE_CONDITIONS); + node3.setType(ConditionsLogicTaskChannelFactory.NAME); // complete 1/2/3/4 expect:8 completeTaskList.put(3L, taskInstance3); postNodes = DagHelper.parsePostNodes(null, skipNodeList, dag, completeTaskList); @@ -308,34 +454,117 @@ public class DagHelperTest { } @Test - public void testSwitchPostNode() throws IOException { - DAG dag = generateDag2(); + public void testSwitchPostNode() { + List taskNodeList = new ArrayList<>(); + + TaskNode node = new TaskNode(); + node.setId("0"); + node.setName("0"); + node.setCode(0); + node.setType("SHELL"); + taskNodeList.add(node); + + TaskNode node1 = new TaskNode(); + node1.setId("1"); + node1.setName("1"); + node1.setCode(1); + node1.setType(SwitchLogicTaskChannelFactory.NAME); + SwitchParameters switchParameters = new SwitchParameters(); + node1.setParams(JSONUtils.toJsonString(switchParameters)); + taskNodeList.add(node1); + + TaskNode node2 = new TaskNode(); + node2.setId("2"); + node2.setName("2"); + node2.setCode(2); + node2.setType("SHELL"); + List dep2 = new ArrayList<>(); + dep2.add("1"); + node2.setPreTasks(JSONUtils.toJsonString(dep2)); + taskNodeList.add(node2); + + TaskNode node4 = new TaskNode(); + node4.setId("4"); + node4.setName("4"); + node4.setCode(4); + node4.setType("SHELL"); + List dep4 = new ArrayList<>(); + dep4.add("1"); + node4.setPreTasks(JSONUtils.toJsonString(dep4)); + taskNodeList.add(node4); + + TaskNode node5 = new TaskNode(); + node5.setId("5"); + node5.setName("5"); + node5.setCode(5); + node5.setType("SHELL"); + List dep5 = new ArrayList<>(); + dep5.add(1L); + node5.setPreTasks(JSONUtils.toJsonString(dep5)); + taskNodeList.add(node5); + + TaskNode node6 = new TaskNode(); + node5.setId("6"); + node5.setName("6"); + node5.setCode(6); + node5.setType("SHELL"); + List dep6 = new ArrayList<>(); + dep5.add(2L); + dep5.add(4L); + node5.setPreTasks(JSONUtils.toJsonString(dep6)); + taskNodeList.add(node6); + + List startNodes = new ArrayList<>(); + List recoveryNodes = new ArrayList<>(); + + // 0 + // 1->2->6 + // 1->4->6 + // 1->5 + List destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList, + startNodes, recoveryNodes, TaskDependType.TASK_POST); + List taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList); + ProcessDag processDag = new ProcessDag(); + processDag.setEdges(taskNodeRelations); + processDag.setNodes(destTaskNodeList); + + DAG dag = DagHelper.buildDagGraph(processDag); Map skipTaskNodeList = new HashMap<>(); Map completeTaskList = new HashMap<>(); - completeTaskList.put(0l, new TaskInstance()); + completeTaskList.put(0L, new TaskInstance()); TaskInstance taskInstance = new TaskInstance(); taskInstance.setState(TaskExecutionStatus.SUCCESS); - taskInstance.setTaskCode(1l); - Map taskParamsMap = new HashMap<>(); - taskParamsMap.put(Constants.SWITCH_RESULT, ""); - taskInstance.setTaskParams(JSONUtils.toJsonString(taskParamsMap)); - taskInstance.setSwitchDependency(getSwitchNode()); + taskInstance.setTaskCode(1L); + taskInstance.setTaskType(SwitchLogicTaskChannelFactory.NAME); + switchParameters = SwitchParameters.builder() + .nextBranch(5L) + .switchResult(SwitchParameters.SwitchResult.builder() + .dependTaskList(Lists.newArrayList( + new SwitchResultVo("", 2L), + new SwitchResultVo("", 4L))) + .nextNode(5L) + .build()) + .build(); + taskInstance.setTaskParams(JSONUtils.toJsonString(switchParameters)); completeTaskList.put(1l, taskInstance); - DagHelper.skipTaskNode4Switch(dag.getNode(1l), skipTaskNodeList, completeTaskList, dag); + List nextBranch = DagHelper.skipTaskNode4Switch(skipTaskNodeList, taskInstance, dag); Assertions.assertNotNull(skipTaskNodeList.get(2L)); - Assertions.assertEquals(1, skipTaskNodeList.size()); + Assertions.assertNotNull(skipTaskNodeList.get(4L)); + Assertions.assertEquals(2, skipTaskNodeList.size()); + Truth.assertThat(nextBranch).containsExactly(5L); } + /** * process: * 1->2->3->5->7 * 4->3->6 * 1->2->8->5->7 * DAG graph: - * 4 -> -> 6 - * \ / + * 4 -> -> 6 + * \ / * 1 -> 2 -> 3 -> 5 -> 7 - * \ / - * -> 8 -> + * \ / + * -> 8 -> * * @return dag * @throws JsonProcessingException if error throws JsonProcessingException @@ -429,108 +658,6 @@ public class DagHelperTest { return DagHelper.buildDagGraph(processDag); } - /** - * DAG graph: - * -> 2-> - * / \ - * / \ - * 0->1(switch)->5 6 - * \ / - * \ / - * -> 4-> - * - * @return dag - * @throws JsonProcessingException if error throws JsonProcessingException - */ - private DAG generateDag2() throws IOException { - List taskNodeList = new ArrayList<>(); - - TaskNode node = new TaskNode(); - node.setId("0"); - node.setName("0"); - node.setCode(0); - node.setType("SHELL"); - taskNodeList.add(node); - - TaskNode node1 = new TaskNode(); - node1.setId("1"); - node1.setName("1"); - node1.setCode(1); - node1.setType("switch"); - node1.setDependence(JSONUtils.toJsonString(getSwitchNode())); - taskNodeList.add(node1); - - TaskNode node2 = new TaskNode(); - node2.setId("2"); - node2.setName("2"); - node2.setCode(2); - node2.setType("SHELL"); - List dep2 = new ArrayList<>(); - dep2.add("1"); - node2.setPreTasks(JSONUtils.toJsonString(dep2)); - taskNodeList.add(node2); - - TaskNode node4 = new TaskNode(); - node4.setId("4"); - node4.setName("4"); - node4.setCode(4); - node4.setType("SHELL"); - List dep4 = new ArrayList<>(); - dep4.add("1"); - node4.setPreTasks(JSONUtils.toJsonString(dep4)); - taskNodeList.add(node4); - - TaskNode node5 = new TaskNode(); - node5.setId("5"); - node5.setName("5"); - node5.setCode(5); - node5.setType("SHELL"); - List dep5 = new ArrayList<>(); - dep5.add(1L); - node5.setPreTasks(JSONUtils.toJsonString(dep5)); - taskNodeList.add(node5); - - TaskNode node6 = new TaskNode(); - node5.setId("6"); - node5.setName("6"); - node5.setCode(6); - node5.setType("SHELL"); - List dep6 = new ArrayList<>(); - dep5.add(2L); - dep5.add(4L); - node5.setPreTasks(JSONUtils.toJsonString(dep6)); - taskNodeList.add(node6); - - List startNodes = new ArrayList<>(); - List recoveryNodes = new ArrayList<>(); - List destTaskNodeList = DagHelper.generateFlowNodeListByStartNode(taskNodeList, - startNodes, recoveryNodes, TaskDependType.TASK_POST); - List taskNodeRelations = DagHelper.generateRelationListByFlowNodes(destTaskNodeList); - ProcessDag processDag = new ProcessDag(); - processDag.setEdges(taskNodeRelations); - processDag.setNodes(destTaskNodeList); - return DagHelper.buildDagGraph(processDag); - } - - private SwitchParameters getSwitchNode() { - SwitchParameters conditionsParameters = new SwitchParameters(); - SwitchResultVo switchResultVo1 = new SwitchResultVo(); - switchResultVo1.setCondition(" 2 == 1"); - switchResultVo1.setNextNode(2L); - SwitchResultVo switchResultVo2 = new SwitchResultVo(); - switchResultVo2.setCondition(" 2 == 2"); - switchResultVo2.setNextNode(4L); - List list = new ArrayList<>(); - list.add(switchResultVo1); - list.add(switchResultVo2); - conditionsParameters.setDependTaskList(list); - conditionsParameters.setNextNode(5L); - conditionsParameters.setRelation("AND"); - conditionsParameters.setResultConditionLocation(1); - // in: AND(AND(1 is SUCCESS)) - return conditionsParameters; - } - @Test public void testBuildDagGraph() { String shellJson = diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java index 11d2549f00..e9a95eee11 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java @@ -78,8 +78,11 @@ public class HdfsStorageOperator extends AbstractStorageOperator implements Clos private void initHdfsPath() { Path path = new Path(resourceBaseAbsolutePath); if (!fs.exists(path)) { - fs.mkdirs(path); - log.info("Create hdfs path: {}", path); + if (!fs.mkdirs(path)) { + log.info("Create hdfs path: {} failed", path); + } else { + log.error("Create hdfs path: {} success", path); + } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml index b115a1b6d1..7c8f24c717 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml @@ -297,5 +297,10 @@ org.projectlombok lombok + + org.junit.jupiter + junit-jupiter + test + diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ILogicTaskChannel.java similarity index 70% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ILogicTaskChannel.java index 5fc7c5073f..7b9766f57d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ILogicTaskChannel.java @@ -15,21 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.enums; - -public enum BlockingOpportunity { - - BLOCKING_ON_SUCCESS("BlockingOnSuccess"), - BLOCKING_ON_FAILED("BlockingOnFailed"); - - private final String desc; - - BlockingOpportunity(String desc) { - this.desc = desc; - } - - public String getDesc() { - return desc; - } +package org.apache.dolphinscheduler.plugin.task.api; +/** + * Used to mark a task channel as a logic task channel, the logic task channel is a special task channel that will be executed at master. + */ +public interface ILogicTaskChannel extends TaskChannel { } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannel.java index 77abae5047..89fa2a07da 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannel.java @@ -18,19 +18,11 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public interface TaskChannel { - void cancelApplication(boolean status); - - // todo: return ITask AbstractTask createTask(TaskExecutionContext taskRequest); - // todo: return IParameters - AbstractParameters parseParameters(ParametersNode parametersNode); - - ResourceParametersHelper getResources(String parameters); + AbstractParameters parseParameters(String taskParams); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java index f7719b4728..7176f1e82f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java @@ -17,16 +17,19 @@ package org.apache.dolphinscheduler.plugin.task.api; -import org.apache.dolphinscheduler.spi.common.UiChannelFactory; import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; -public interface TaskChannelFactory extends UiChannelFactory, PrioritySPI { - - TaskChannel create(); +public interface TaskChannelFactory extends PrioritySPI { default SPIIdentify getIdentify() { - return SPIIdentify.builder().name(getName()).build(); + return SPIIdentify.builder() + .name(getName()) + .build(); } + String getName(); + + TaskChannel create(); + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index 43734416e7..98780a2236 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -362,24 +362,10 @@ public class TaskConstants { */ public static final String DATA_QUALITY_JAR_DIR = "data-quality.jar.dir"; - public static final String TASK_TYPE_CONDITIONS = "CONDITIONS"; - - public static final String TASK_TYPE_SWITCH = "SWITCH"; - - public static final String TASK_TYPE_SUB_PROCESS = "SUB_PROCESS"; - - public static final String TASK_TYPE_DYNAMIC = "DYNAMIC"; - - public static final String TASK_TYPE_DEPENDENT = "DEPENDENT"; - - public static final String TASK_TYPE_SQL = "SQL"; - public static final String TASK_TYPE_DATA_QUALITY = "DATA_QUALITY"; public static final Set TASK_TYPE_SET_K8S = Sets.newHashSet("K8S", "KUBEFLOW"); - public static final String TASK_TYPE_BLOCKING = "BLOCKING"; - /** * azure config */ @@ -389,13 +375,6 @@ public class TaskConstants { public static final String AZURE_SECRET_TENANT_ID = "resource.azure.tenant.id"; public static final String QUERY_INTERVAL = "resource.query.interval"; - /** - * aws config - */ - public static final String AWS_ACCESS_KEY_ID = "resource.aws.access.key.id"; - public static final String AWS_SECRET_ACCESS_KEY = "resource.aws.secret.access.key"; - public static final String AWS_REGION = "resource.aws.region"; - /** * alibaba cloud config */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java index 938aa77459..7e61fe4dff 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManager.java @@ -17,21 +17,13 @@ package org.apache.dolphinscheduler.plugin.task.api; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import static com.google.common.base.Preconditions.checkNotNull; + import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; @@ -39,15 +31,15 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class TaskPluginManager { - private static final Map taskChannelFactoryMap = new HashMap<>(); private static final Map taskChannelMap = new HashMap<>(); private static final AtomicBoolean loadedFlag = new AtomicBoolean(false); - /** - * Load task plugins from classpath. - */ - public static void loadPlugin() { + static { + loadTaskPlugin(); + } + + public static void loadTaskPlugin() { if (!loadedFlag.compareAndSet(false, true)) { log.warn("The task plugin has already been loaded"); return; @@ -59,7 +51,6 @@ public class TaskPluginManager { log.info("Registering task plugin: {} - {}", factoryName, factory.getClass().getSimpleName()); - taskChannelFactoryMap.put(factoryName, factory); taskChannelMap.put(factoryName, factory.create()); log.info("Registered task plugin: {} - {}", factoryName, factory.getClass().getSimpleName()); @@ -67,48 +58,49 @@ public class TaskPluginManager { } - public static Map getTaskChannelMap() { - return Collections.unmodifiableMap(taskChannelMap); - } - - public static Map getTaskChannelFactoryMap() { - return Collections.unmodifiableMap(taskChannelFactoryMap); - } - + /** + * Get the TaskChannel by type, if the TaskChannel is not found, will throw + * @param type task type, cannot be null + * @throws IllegalArgumentException if the TaskChannel is not found + */ public static TaskChannel getTaskChannel(String type) { - return getTaskChannelMap().get(type); + checkNotNull(type, "type cannot be null"); + TaskChannel taskChannel = taskChannelMap.get(type); + if (taskChannel == null) { + throw new IllegalArgumentException("Cannot find TaskChannel for : " + type); + } + return taskChannel; } - public static boolean checkTaskParameters(ParametersNode parametersNode) { - AbstractParameters abstractParameters = getParameters(parametersNode); - return abstractParameters != null && abstractParameters.checkParameters(); + /** + * Check if the task parameters is validated + * @param taskType task type, cannot be null + * @param taskParams task parameters + * @return true if the task parameters is validated, otherwise false + * @throws IllegalArgumentException if the TaskChannel is not found + * @throws IllegalArgumentException if cannot deserialize the task parameters + */ + public static boolean checkTaskParameters(String taskType, String taskParams) { + AbstractParameters abstractParameters = parseTaskParameters(taskType, taskParams); + return abstractParameters.checkParameters(); } - public static AbstractParameters getParameters(ParametersNode parametersNode) { - String taskType = parametersNode.getTaskType(); - if (Objects.isNull(taskType)) { - return null; - } - switch (taskType) { - case TaskConstants.TASK_TYPE_CONDITIONS: - return JSONUtils.parseObject(parametersNode.getTaskParams(), ConditionsParameters.class); - case TaskConstants.TASK_TYPE_SWITCH: - return JSONUtils.parseObject(parametersNode.getTaskParams(), SwitchParameters.class); - case TaskConstants.TASK_TYPE_SUB_PROCESS: - return JSONUtils.parseObject(parametersNode.getTaskParams(), SubProcessParameters.class); - case TaskConstants.TASK_TYPE_DEPENDENT: - return JSONUtils.parseObject(parametersNode.getTaskParams(), DependentParameters.class); - case TaskConstants.TASK_TYPE_BLOCKING: - return JSONUtils.parseObject(parametersNode.getTaskParams(), BlockingParameters.class); - case TaskConstants.TASK_TYPE_DYNAMIC: - return JSONUtils.parseObject(parametersNode.getTaskParams(), DynamicParameters.class); - default: - TaskChannel taskChannel = getTaskChannelMap().get(taskType); - if (Objects.isNull(taskChannel)) { - return null; - } - return taskChannel.parseParameters(parametersNode); + /** + * Parse the task parameters + * @param taskType task type, cannot be null + * @param taskParams task parameters + * @return AbstractParameters + * @throws IllegalArgumentException if the TaskChannel is not found + * @throws IllegalArgumentException if cannot deserialize the task parameters + */ + public static AbstractParameters parseTaskParameters(String taskType, String taskParams) { + checkNotNull(taskType, "taskType cannot be null"); + TaskChannel taskChannel = getTaskChannel(taskType); + AbstractParameters abstractParameters = taskChannel.parseParameters(taskParams); + if (abstractParameters == null) { + throw new IllegalArgumentException("Cannot parse task parameters: " + taskParams + " for : " + taskType); } + return abstractParameters; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DynamicInputParameter.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DynamicInputParameter.java index 40f85087cf..cd5c1005c4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DynamicInputParameter.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DynamicInputParameter.java @@ -17,12 +17,16 @@ package org.apache.dolphinscheduler.plugin.task.api.model; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; @Data +@Builder @NoArgsConstructor +@AllArgsConstructor public class DynamicInputParameter { @NonNull diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java index 3af8fad195..51d53080eb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import java.util.ArrayList; -import java.util.List; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -30,15 +27,5 @@ import lombok.NoArgsConstructor; public class SwitchResultVo { private String condition; - private List nextNode; - - public void setNextNode(Object nextNode) { - if (nextNode instanceof Long) { - List nextNodeList = new ArrayList<>(); - nextNodeList.add((Long) nextNode); - this.nextNode = nextNodeList; - } else { - this.nextNode = (ArrayList) nextNode; - } - } + private Long nextNode; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java deleted file mode 100644 index 2942a08c30..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/BlockingParameters.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.parameters; - -import org.apache.commons.lang3.StringUtils; - -public class BlockingParameters extends AbstractParameters { - - // condition of blocking: BlockingOnFailed or BlockingOnSuccess - private String blockingOpportunity; - - // if true, alert when blocking, otherwise do nothing - - private boolean isAlertWhenBlocking; - - @Override - public boolean checkParameters() { - return !StringUtils.isEmpty(blockingOpportunity); - } - - public String getBlockingOpportunity() { - return blockingOpportunity; - } - - public void setBlockingCondition(String blockingOpportunity) { - this.blockingOpportunity = blockingOpportunity; - } - - public boolean isAlertWhenBlocking() { - return isAlertWhenBlocking; - } - - public void setAlertWhenBlocking(boolean alertWhenBlocking) { - isAlertWhenBlocking = alertWhenBlocking; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java index 59b0d08fc5..8ad567578c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java @@ -19,36 +19,59 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import java.util.ArrayList; +import org.apache.commons.collections4.CollectionUtils; + import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class ConditionsParameters extends AbstractParameters { - // depend node list and state, only need task name - private List dependTaskList; - private DependentRelation relation; + private ConditionDependency dependence; - private boolean conditionSuccess; + private ConditionResult conditionResult; @Override public boolean checkParameters() { + if (dependence == null || CollectionUtils.isEmpty(dependence.getDependTaskList())) { + return false; + } + if (conditionResult == null || CollectionUtils.isEmpty(conditionResult.getSuccessNode()) + || CollectionUtils.isEmpty(conditionResult.getFailedNode())) { + return false; + } return true; } - @Override - public List getResourceFilesList() { - return new ArrayList<>(); + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class ConditionDependency { + + private List dependTaskList; + + private DependentRelation relation; } @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor public static class ConditionResult { + private boolean conditionSuccess; + private List successNode; + private List failedNode; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java index 1e648d5775..c5f9ed916f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java @@ -20,28 +20,55 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; +import org.apache.commons.collections4.CollectionUtils; + import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; @Data @EqualsAndHashCode(callSuper = true) public class DependentParameters extends AbstractParameters { - private List dependTaskList; - private DependentRelation relation; - /** Time unit is second */ - private Integer checkInterval; - private DependentFailurePolicyEnum failurePolicy; - /** Time unit is minutes */ - private Integer failureWaitingTime; + private Dependence dependence; @Override public boolean checkParameters() { + if (dependence == null) { + return false; + } + if (CollectionUtils.isEmpty(dependence.getDependTaskList())) { + return false; + } + if (dependence.getCheckInterval() != null && dependence.getCheckInterval() <= 0) { + return false; + } return true; } + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class Dependence { + + private List dependTaskList; + private DependentRelation relation; + /** + * Time unit is second + */ + private Integer checkInterval; + private DependentFailurePolicyEnum failurePolicy; + /** + * Time unit is minutes + */ + private Integer failureWaitingTime; + } + /** * the dependent task failure policy. */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DynamicParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DynamicParameters.java index e918db3b18..2a594655f9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DynamicParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DynamicParameters.java @@ -21,9 +21,15 @@ import org.apache.dolphinscheduler.plugin.task.api.model.DynamicInputParameter; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class DynamicParameters extends AbstractParameters { /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java deleted file mode 100644 index b6eb88a68a..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.task.api.parameters; - -/** - * TODO

Need to optimize, why there are multiple task parameter variables:taskParams,dependence,switchResult

- * - */ -public class ParametersNode { - - private String taskType; - - private String taskParams; - - private String dependence; - - private String switchResult; - - public static ParametersNode.ParametersNodeBuilder builder() { - return new ParametersNode.ParametersNodeBuilder(); - } - - public static class ParametersNodeBuilder { - - private String taskType; - - private String taskParams; - - private String dependence; - - private String switchResult; - - public ParametersNodeBuilder taskType(String taskType) { - this.taskType = taskType; - return this; - } - - public ParametersNodeBuilder taskParams(String taskParams) { - this.taskParams = taskParams; - return this; - } - - public ParametersNodeBuilder dependence(String dependence) { - this.dependence = dependence; - return this; - } - - public ParametersNodeBuilder switchResult(String switchResult) { - this.switchResult = switchResult; - return this; - } - - public ParametersNode build() { - return new ParametersNode(this.taskType, this.taskParams, this.dependence, this.switchResult); - } - - } - - public ParametersNode() { - - } - - public ParametersNode(String taskType, String taskParams, String dependence, String switchResult) { - this.taskType = taskType; - this.taskParams = taskParams; - this.dependence = dependence; - this.switchResult = switchResult; - } - - public String getTaskType() { - return taskType; - } - - public void setTaskType(String taskType) { - this.taskType = taskType; - } - - public String getTaskParams() { - return taskParams; - } - - public void setTaskParams(String taskParams) { - this.taskParams = taskParams; - } - - public String getDependence() { - return dependence; - } - - public void setDependence(String dependence) { - this.dependence = dependence; - } - - public String getSwitchResult() { - return switchResult; - } - - public void setSwitchResult(String switchResult) { - this.switchResult = switchResult; - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java index 3368313935..655ba7540c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; public class SubProcessParameters extends AbstractParameters { - /** - * process definition id - */ private long processDefinitionCode; public void setProcessDefinitionCode(long processDefinitionCode) { @@ -34,7 +31,7 @@ public class SubProcessParameters extends AbstractParameters { @Override public boolean checkParameters() { - return this.processDefinitionCode != 0; + return this.processDefinitionCode > 0; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java index a1928bbe16..655a86f9e0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java @@ -17,69 +17,56 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; -import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; -import java.util.ArrayList; +import org.apache.commons.collections4.CollectionUtils; + import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class SwitchParameters extends AbstractParameters { - private DependentRelation dependRelation; - private String relation; - private List nextNode; + // due to history reasons, the field name is switchResult + private SwitchResult switchResult; + + // The next branch which should be executed after the switch logic task executed. + private Long nextBranch; @Override public boolean checkParameters() { + if (switchResult == null) { + return false; + } + if (CollectionUtils.isEmpty(switchResult.getDependTaskList()) && switchResult.getNextNode() == null) { + return false; + } + for (SwitchResultVo switchResultVo : switchResult.getDependTaskList()) { + if (switchResultVo == null || switchResultVo.getNextNode() == null) { + return false; + } + } return true; } - private int resultConditionLocation; - private List dependTaskList; + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class SwitchResult { - public DependentRelation getDependRelation() { - return dependRelation; - } - - public void setDependRelation(DependentRelation dependRelation) { - this.dependRelation = dependRelation; - } + // switch condition + private List dependTaskList; - public int getResultConditionLocation() { - return resultConditionLocation; + // default branch node code in switch task + private Long nextNode; } - public void setResultConditionLocation(int resultConditionLocation) { - this.resultConditionLocation = resultConditionLocation; - } - - public String getRelation() { - return relation; - } - - public void setRelation(String relation) { - this.relation = relation; - } - - public List getDependTaskList() { - return dependTaskList; - } - - public void setDependTaskList(List dependTaskList) { - this.dependTaskList = dependTaskList; - } - - public List getNextNode() { - return nextNode; - } - - public void setNextNode(Object nextNode) { - if (nextNode instanceof Long) { - List nextNodeList = new ArrayList<>(); - nextNodeList.add((Long) nextNode); - this.nextNode = nextNodeList; - } else { - this.nextNode = (ArrayList) nextNode; - } - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java index 6c35b3cec4..bc0107a11e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java @@ -17,13 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.api.stream; -import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; public interface StreamTaskChannel extends TaskChannel { - AbstractTask pauseTask(TaskExecutionContext taskExecutionContext); - - AbstractTask recoverTask(TaskExecutionContext taskExecutionContext); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/AbstractLogicTaskChannel.java similarity index 58% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/AbstractLogicTaskChannel.java index 1504fdd367..b0bcdc8767 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/AbstractLogicTaskChannel.java @@ -15,47 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.entity; +package org.apache.dolphinscheduler.plugin.task.api.task; -import java.util.Date; +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.ILogicTaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import lombok.Data; +public abstract class AbstractLogicTaskChannel implements ILogicTaskChannel { -@Data -public class WorkerServer { + @Override + public AbstractTask createTask(TaskExecutionContext taskRequest) { + throw new UnsupportedOperationException("createTask is not supported"); + } - /** - * id - */ - private int id; - - /** - * host - */ - private String host; - - /** - * port - */ - private int port; - - /** - * zookeeper directory - */ - private String zkDirectory; - - /** - * resource info - */ - private String resInfo; - - /** - * create time - */ - private Date createTime; - - /** - * last heart beat time - */ - private Date lastHeartbeatTime; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannel.java new file mode 100644 index 0000000000..976c658303 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannel.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; + +public class ConditionsLogicTaskChannel extends AbstractLogicTaskChannel { + + @Override + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, ConditionsParameters.class); + } + +} diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannelFactory.java similarity index 55% rename from dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannelFactory.java index 8b89215049..fd63ea5039 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/ConditionsLogicTaskChannelFactory.java @@ -15,29 +15,25 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.spi.common; +package org.apache.dolphinscheduler.plugin.task.api.task; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import java.util.List; +import com.google.auto.service.AutoService; -public interface UiChannelFactory { +@AutoService(TaskChannelFactory.class) +public class ConditionsLogicTaskChannelFactory implements TaskChannelFactory { - /** - * plugin name - * Must be UNIQUE . - * This alert plugin name eg: email , message ... - * Name can often be displayed on the page ui eg : email , message , MR , spark , hive ... - * - * @return this alert plugin name - */ - String getName(); + public static final String NAME = "CONDITIONS"; - /** - * Returns the configurable parameters that this plugin needs to display on the web ui - * - * @return this alert plugin params - */ - List getParams(); + @Override + public String getName() { + return NAME; + } + @Override + public TaskChannel create() { + return new ConditionsLogicTaskChannel(); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannel.java new file mode 100644 index 0000000000..04045f0f4b --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; + +public class DependentLogicTaskChannel extends AbstractLogicTaskChannel { + + @Override + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DependentParameters.class); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannelFactory.java new file mode 100644 index 0000000000..332a1c6c42 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DependentLogicTaskChannelFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class DependentLogicTaskChannelFactory implements TaskChannelFactory { + + public static final String NAME = "DEPENDENT"; + @Override + public String getName() { + return NAME; + } + + @Override + public TaskChannel create() { + return new DependentLogicTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannel.java new file mode 100644 index 0000000000..d9f066d773 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters; + +public class DynamicLogicTaskChannel extends AbstractLogicTaskChannel { + + @Override + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DynamicParameters.class); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannelFactory.java new file mode 100644 index 0000000000..0164e3ba53 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/DynamicLogicTaskChannelFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class DynamicLogicTaskChannelFactory implements TaskChannelFactory { + + public static final String NAME = "DYNAMIC"; + @Override + public String getName() { + return NAME; + } + + @Override + public TaskChannel create() { + return new DynamicLogicTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannel.java new file mode 100644 index 0000000000..354be54986 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; + +public class SubWorkflowLogicTaskChannel extends AbstractLogicTaskChannel { + + @Override + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SubProcessParameters.class); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannelFactory.java new file mode 100644 index 0000000000..06bb6c5ebe --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SubWorkflowLogicTaskChannelFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class SubWorkflowLogicTaskChannelFactory implements TaskChannelFactory { + + public static final String NAME = "SUB_PROCESS"; + + @Override + public String getName() { + return NAME; + } + + @Override + public TaskChannel create() { + return new SubWorkflowLogicTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannel.java new file mode 100644 index 0000000000..6d549bcb9b --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; + +public class SwitchLogicTaskChannel extends AbstractLogicTaskChannel { + + @Override + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SwitchParameters.class); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannelFactory.java new file mode 100644 index 0000000000..3f5b254ca6 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/task/SwitchLogicTaskChannelFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.task; + +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(TaskChannelFactory.class) +public class SwitchLogicTaskChannelFactory implements TaskChannelFactory { + + public static final String NAME = "SWITCH"; + + @Override + public String getName() { + return NAME; + } + + @Override + public TaskChannel create() { + return new SwitchLogicTaskChannel(); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskTypeUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskTypeUtils.java new file mode 100644 index 0000000000..cf1f2022ea --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskTypeUtils.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.task.api.utils; + +import org.apache.dolphinscheduler.plugin.task.api.ILogicTaskChannel; +import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.DynamicLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class TaskTypeUtils { + + public boolean isSwitchTask(String taskType) { + return SwitchLogicTaskChannelFactory.NAME.equals(taskType); + } + + public boolean isConditionTask(String taskType) { + return ConditionsLogicTaskChannelFactory.NAME.equals(taskType); + } + + public boolean isSubWorkflowTask(String taskType) { + return SubWorkflowLogicTaskChannelFactory.NAME.equals(taskType); + } + + public boolean isDependentTask(String taskType) { + return SubWorkflowLogicTaskChannelFactory.NAME.equals(taskType); + } + + public boolean isDynamicTask(String taskType) { + return DynamicLogicTaskChannelFactory.NAME.equals(taskType); + } + + public boolean isLogicTask(String taskType) { + return TaskPluginManager.getTaskChannel(taskType) instanceof ILogicTaskChannel; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java deleted file mode 100644 index 606058d40f..0000000000 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.plugin.task.api.utils; - -import java.util.Set; - -import lombok.experimental.UtilityClass; - -import com.google.common.collect.Sets; - -@UtilityClass -public class TaskUtils { - - private final String blockingLogicTask = "BLOCKING"; - private final String conditionLogicTask = "CONDITIONS"; - - private final String dependentLogicTask = "DEPENDENT"; - private final String subWorkflowLogicTask = "SUB_PROCESS"; - private final String switchLogicTask = "SWITCH"; - private final String dynamicLogicTask = "DYNAMIC"; - - // todo: Add to SPI - private final Set MASTER_TASK_TYPES = Sets.newHashSet( - blockingLogicTask, - conditionLogicTask, - dependentLogicTask, - subWorkflowLogicTask, - switchLogicTask, - dynamicLogicTask); - - // todo: add to task plugin spi - public boolean isLogicTask(String taskType) { - return MASTER_TASK_TYPES.contains(taskType); - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManagerTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManagerTest.java new file mode 100644 index 0000000000..a380af488f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginManagerTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api; + +import static com.google.common.truth.Truth.assertThat; + +import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.DynamicLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory; +import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class TaskPluginManagerTest { + + @ParameterizedTest + @ValueSource(strings = { + ConditionsLogicTaskChannelFactory.NAME, + DependentLogicTaskChannelFactory.NAME, + DynamicLogicTaskChannelFactory.NAME, + SubWorkflowLogicTaskChannelFactory.NAME, + SwitchLogicTaskChannelFactory.NAME}) + void testGetTaskChannel_logicTaskChannel(String type) { + assertThat(TaskPluginManager.getTaskChannel(type)).isNotNull(); + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannel.java index 02063d07c1..cce2e28090 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannel.java @@ -22,31 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; -/** - * chunjun task channel - */ public class ChunJunTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskExecutionContext) { return new ChunJunTask(taskExecutionContext); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), ChunJunParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, ChunJunParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, ChunJunParameters.class).getResources(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannelFactory.java index 06a0de36f1..b1465d723a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.chunjun; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -47,13 +44,4 @@ public class ChunJunTaskChannelFactory implements TaskChannelFactory { return "CHUNJUN"; } - /** - * Returns the configurable parameters that this plugin needs to display on the web ui - * - * @return this plugin params - */ - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannel.java index d8c099bb09..b5062a3d4c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannel.java @@ -21,27 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DatafactoryTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - } - @Override public DatafactoryTask createTask(TaskExecutionContext taskRequest) { return new DatafactoryTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DatafactoryParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DatafactoryParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannelFactory.java index e9dce1ac96..0ceb0ea2ed 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datafactory/src/main/java/org/apache/dolphinscheduler/plugin/task/datafactory/DatafactoryTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.datafactory; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -39,8 +35,4 @@ public class DatafactoryTaskChannelFactory implements TaskChannelFactory { return "DATA_FACTORY"; } - @Override - public List getParams() { - return Collections.emptyList(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java index b14596978c..1608de87af 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java @@ -22,29 +22,18 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DataQualityTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new DataQualityTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DataQualityParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DataQualityParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannelFactory.java index f56d65723f..7ad56daf13 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dq; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class DataQualityTaskChannelFactory implements TaskChannelFactory { return "DATA_QUALITY"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new DataQualityTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java index 1d4351caae..0e390c757e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java @@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DatasyncTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public DatasyncTask createTask(TaskExecutionContext taskRequest) { return new DatasyncTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DatasyncParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DatasyncParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java index 5fac501bea..770b07d3bb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.datasync; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -39,8 +35,4 @@ public class DatasyncTaskChannelFactory implements TaskChannelFactory { return "DATASYNC"; } - @Override - public List getParams() { - return Collections.emptyList(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java index 5585b3d0b7..fd93e9c372 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DataxTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new DataxTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DataxParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DataxParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, DataxParameters.class).getResources(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java index 332d6bf708..2416081acf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.datax; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class DataxTaskChannelFactory implements TaskChannelFactory { return "DATAX"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new DataxTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java index 83846ff0fc..681d965fa8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DinkyTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - // nothing to do - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new DinkyTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DinkyParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DinkyParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java index ddaa53d233..29f35b65e0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dinky; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.ArrayList; -import java.util.List; import com.google.auto.service.AutoService; @@ -34,11 +30,6 @@ public class DinkyTaskChannelFactory implements TaskChannelFactory { return "DINKY"; } - @Override - public List getParams() { - return new ArrayList<>(); - } - @Override public TaskChannel create() { return new DinkyTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannel.java index 45f26c5c1e..cb8d8893bd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DmsTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public DmsTask createTask(TaskExecutionContext taskRequest) { return new DmsTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DmsParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DmsParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java index a8564ff8a1..079e630d0c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dms; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -39,8 +35,4 @@ public class DmsTaskChannelFactory implements TaskChannelFactory { return "DMS"; } - @Override - public List getParams() { - return Collections.emptyList(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java index cf78414918..06f4421ca4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class DvcTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public DvcTask createTask(TaskExecutionContext taskRequest) { return new DvcTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), DvcParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, DvcParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java index a29ffded5b..b07612ad2e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java @@ -19,14 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.dvc; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.ParamsOptions; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.params.base.Validate; -import org.apache.dolphinscheduler.spi.params.input.InputParam; -import org.apache.dolphinscheduler.spi.params.radio.RadioParam; - -import java.util.ArrayList; -import java.util.List; import com.google.auto.service.AutoService; @@ -43,19 +35,4 @@ public class DvcTaskChannelFactory implements TaskChannelFactory { return "DVC"; } - @Override - public List getParams() { - List paramsList = new ArrayList<>(); - - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") - .addValidate(Validate.newBuilder().setRequired(true).build()).build(); - - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") - .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) - .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); - - paramsList.add(nodeName); - paramsList.add(runFlag); - return paramsList; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java index 8e42eb600f..80bb271ba4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java @@ -22,16 +22,9 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class EmrTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - // no need - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { EmrParameters emrParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), EmrParameters.class); @@ -46,12 +39,8 @@ public class EmrTaskChannel implements TaskChannel { } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), EmrParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, EmrParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java index cad5583f07..606b0a64d8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.emr; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -34,11 +30,6 @@ public class EmrTaskChannelFactory implements TaskChannelFactory { return "EMR"; } - @Override - public List getParams() { - return Collections.emptyList(); - } - @Override public TaskChannel create() { return new EmrTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java index 50f0984021..fd349e2abb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java @@ -18,42 +18,20 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTaskChannel; public class FlinkStreamTaskChannel implements StreamTaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public FlinkStreamTask createTask(TaskExecutionContext taskRequest) { return new FlinkStreamTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), FlinkStreamParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } - - @Override - public AbstractTask pauseTask(TaskExecutionContext taskExecutionContext) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, FlinkStreamParameters.class); } - @Override - public AbstractTask recoverTask(TaskExecutionContext taskExecutionContext) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java index 475386bf36..c75e7dbfb8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class FlinkStreamTaskChannelFactory implements TaskChannelFactory { return "FLINK_STREAM"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java index ff11983f75..37a003c10c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java @@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class FlinkTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public FlinkTask createTask(TaskExecutionContext taskRequest) { return new FlinkTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), FlinkParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, FlinkParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java index 953f013fce..155e031d04 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class FlinkTaskChannelFactory implements TaskChannelFactory { return "FLINK"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java index d45eae34e1..5397928ec0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class HiveCliTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskExecutionContext) { return new HiveCliTask(taskExecutionContext); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), HiveCliParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, HiveCliParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java index 244ac2eba3..292a027fb4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.hivecli; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class HiveCliTaskChannelFactory implements TaskChannelFactory { return "HIVECLI"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java index b91aa5bcb6..4589f5ac7e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class HttpTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new HttpTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), HttpParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, HttpParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java index efbc254b0d..e7743cfe19 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.http; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class HttpTaskChannelFactory implements TaskChannelFactory { return "HTTP"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new HttpTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java index 710342ed13..26f04f0c40 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannel.java @@ -21,52 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class JavaTaskChannel implements TaskChannel { - /** - * Cancel the mission - * - * @param status - * @return void - **/ - @Override - public void cancelApplication(boolean status) { - - } - - /** - * Create a task - * - * @param taskRequest This parameter is the Echternach of the mission - * @return JavaTask - **/ @Override public JavaTask createTask(TaskExecutionContext taskRequest) { return new JavaTask(taskRequest); } - /** - * Parses Java task parameters - * - * @param parametersNode - * @return: org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters - **/ @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), JavaParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, JavaParameters.class); } - /** - * Gets a list of the resources that the task depends on - * - * @param parameters - * @return ResourceParametersHelper - **/ - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java index e72f62e29e..4db03756d4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.java; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -48,13 +45,4 @@ public class JavaTaskChannelFactory implements TaskChannelFactory { return "JAVA"; } - /** - * Gets the plug-in parameters for the Java task - * - * @return List - **/ - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannel.java index c98625c89e..a93e77a827 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class JupyterTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new JupyterTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), JupyterParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, JupyterParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java index 41bff2bf80..ba2709d388 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.jupyter; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class JupyterTaskChannelFactory implements TaskChannelFactory { return "JUPYTER"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new JupyterTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java index 91c0faf684..b512201edd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java @@ -23,24 +23,12 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class K8sTaskChannel implements TaskChannel { @Override - public void cancelApplication(boolean status) { - - } - - @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), K8sTaskParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, K8sTaskParameters.class).getResources(); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, K8sTaskParameters.class); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java index 4af560b5d2..b767c4bd33 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.k8s; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class K8sTaskChannelFactory implements TaskChannelFactory { return "K8S"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new K8sTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannel.java index 5bfdc9e258..88e388c9f9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannel.java @@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class KubeflowTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public KubeflowTask createTask(TaskExecutionContext taskRequest) { return new KubeflowTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), KubeflowParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, KubeflowParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannelFactory.java index 9481311ce6..1a18cc03b0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-kubeflow/src/main/java/org/apache/dolphinscheduler/plugin/kubeflow/KubeflowTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.kubeflow; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -39,9 +35,4 @@ public class KubeflowTaskChannelFactory implements TaskChannelFactory { return "KUBEFLOW"; } - @Override - public List getParams() { - return Collections.emptyList(); - } - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannel.java index b5d357a45e..d49006dd6d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class LinkisTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public LinkisTask createTask(TaskExecutionContext taskRequest) { return new LinkisTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), LinkisParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, LinkisParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannelFactory.java index 5d2c0d563e..a3bc274234 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-linkis/src/main/java/org/apache/dolphinscheduler/plugin/task/linkis/LinkisTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.linkis; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class LinkisTaskChannelFactory implements TaskChannelFactory { return "LINKIS"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java index 7dfacd0006..a2687c2634 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class MlflowTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public MlflowTask createTask(TaskExecutionContext taskRequest) { return new MlflowTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), MlflowParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, MlflowParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java index dc18814d1d..e4985d42f2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java @@ -19,14 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.mlflow; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.ParamsOptions; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.params.base.Validate; -import org.apache.dolphinscheduler.spi.params.input.InputParam; -import org.apache.dolphinscheduler.spi.params.radio.RadioParam; - -import java.util.ArrayList; -import java.util.List; import com.google.auto.service.AutoService; @@ -43,19 +35,4 @@ public class MlflowTaskChannelFactory implements TaskChannelFactory { return "MLFLOW"; } - @Override - public List getParams() { - List paramsList = new ArrayList<>(); - - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") - .addValidate(Validate.newBuilder().setRequired(true).build()).build(); - - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") - .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) - .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); - - paramsList.add(nodeName); - paramsList.add(runFlag); - return paramsList; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java index 9dc9273769..525376cd9b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class MapReduceTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new MapReduceTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), MapReduceParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, MapReduceParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java index e42737d83e..87951d715b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.mr; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class MapReduceTaskChannelFactory implements TaskChannelFactory { return "MR"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new MapReduceTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java index b4e7ca8e8d..aab2a4b7a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java @@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class OpenmldbTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public OpenmldbTask createTask(TaskExecutionContext taskRequest) { return new OpenmldbTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), OpenmldbParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, OpenmldbParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java index 32e3eaa04d..7a0e8d5a3a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.openmldb; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class OpenmldbTaskChannelFactory implements TaskChannelFactory { return "OPENMLDB"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java index 9a1c20fcd1..112f087a57 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java @@ -22,31 +22,20 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import lombok.extern.slf4j.Slf4j; @Slf4j public class PigeonTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - log.info("pigeon task cancel"); - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new PigeonTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), PigeonParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, PigeonParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java index a7ce316921..584f131a42 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java @@ -19,12 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.params.base.Validate; -import org.apache.dolphinscheduler.spi.params.input.InputParam; - -import java.util.Arrays; -import java.util.List; import com.google.auto.service.AutoService; @@ -41,12 +35,4 @@ public class PigeonTaskChannelFactory implements TaskChannelFactory { return "PIGEON"; } - @Override - public List getParams() { - InputParam webHookParam = - InputParam.newBuilder(PigeonParamsConstants.NAME_TARGET_JOB_NAME, PigeonParamsConstants.TARGET_JOB_NAME) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - return Arrays.asList(webHookParam); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java index ccad7d0c4f..7138cddc89 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class ProcedureTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new ProcedureTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), ProcedureParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, ProcedureParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, ProcedureParameters.class).getResources(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java index a96a082af0..4da62ab5bf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.procedure; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class ProcedureTaskChannelFactory implements TaskChannelFactory { return "PROCEDURE"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new ProcedureTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java index d602d4e162..a7a1131a50 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java @@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class PythonTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public PythonTask createTask(TaskExecutionContext taskRequest) { return new PythonTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), PythonParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, PythonParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java index ddd969f198..214a3a9b25 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.python; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class PythonTaskChannelFactory implements TaskChannelFactory { return "PYTHON"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannel.java index 6d2427e594..5d4f7a63f4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class PytorchTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public PytorchTask createTask(TaskExecutionContext taskRequest) { return new PytorchTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), PytorchParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, PytorchParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java index bf1bda8b2d..7b4fd09b59 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.pytorch; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -39,8 +35,4 @@ public class PytorchTaskChannelFactory implements TaskChannelFactory { return "PYTORCH"; } - @Override - public List getParams() { - return Collections.emptyList(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannel.java index e2baeca3d1..7143aa4ff7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannel.java @@ -21,28 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class RemoteShellTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public RemoteShellTask createTask(TaskExecutionContext taskRequest) { return new RemoteShellTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), RemoteShellParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, RemoteShellParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, RemoteShellParameters.class).getResources(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannelFactory.java index 394a0aae76..2f51f83cc3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-remoteshell/src/main/java/org/apache/dolphinscheduler/plugin/task/remoteshell/RemoteShellTaskChannelFactory.java @@ -19,14 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.remoteshell; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.ParamsOptions; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.params.base.Validate; -import org.apache.dolphinscheduler.spi.params.input.InputParam; -import org.apache.dolphinscheduler.spi.params.radio.RadioParam; - -import java.util.ArrayList; -import java.util.List; import com.google.auto.service.AutoService; @@ -43,23 +35,4 @@ public class RemoteShellTaskChannelFactory implements TaskChannelFactory { return "REMOTESHELL"; } - @Override - public List getParams() { - List paramsList = new ArrayList<>(); - - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); - - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") - .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) - .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)) - .build(); - - paramsList.add(nodeName); - paramsList.add(runFlag); - return paramsList; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannel.java index 4dcdb051e9..926d23df05 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class SagemakerTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public SagemakerTask createTask(TaskExecutionContext taskRequest) { return new SagemakerTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), SagemakerParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, SagemakerParameters.class).getResources(); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SagemakerParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java index ccad195e0e..370f8beacd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java @@ -19,10 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.sagemaker; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.Collections; -import java.util.List; import com.google.auto.service.AutoService; @@ -39,8 +35,4 @@ public class SagemakerTaskChannelFactory implements TaskChannelFactory { return "SAGEMAKER"; } - @Override - public List getParams() { - return Collections.emptyList(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java index ae01d8edd8..9632f8051a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java @@ -25,19 +25,12 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.seatunnel.flink.SeatunnelFlinkTask; import org.apache.dolphinscheduler.plugin.task.seatunnel.self.SeatunnelEngineTask; import org.apache.dolphinscheduler.plugin.task.seatunnel.spark.SeatunnelSparkTask; public class SeatunnelTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public SeatunnelTask createTask(TaskExecutionContext taskRequest) { SeatunnelParameters seatunnelParameters = @@ -57,13 +50,8 @@ public class SeatunnelTaskChannel implements TaskChannel { } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), SeatunnelParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SeatunnelParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannelFactory.java index 2763c00d50..ab7b3f0cf1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.seatunnel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -38,8 +35,4 @@ public class SeatunnelTaskChannelFactory implements TaskChannelFactory { return "SEATUNNEL"; } - @Override - public List getParams() { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java index 12f0263aa1..26c1019700 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannel.java @@ -21,29 +21,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class ShellTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public ShellTask createTask(TaskExecutionContext taskRequest) { return new ShellTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), ShellParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, ShellParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java index 801826ffd1..560bae5b1d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java @@ -19,20 +19,14 @@ package org.apache.dolphinscheduler.plugin.task.shell; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.ParamsOptions; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.params.base.Validate; -import org.apache.dolphinscheduler.spi.params.input.InputParam; -import org.apache.dolphinscheduler.spi.params.radio.RadioParam; - -import java.util.ArrayList; -import java.util.List; import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class ShellTaskChannelFactory implements TaskChannelFactory { + public static final String NAME = "SHELL"; + @Override public TaskChannel create() { return new ShellTaskChannel(); @@ -40,26 +34,7 @@ public class ShellTaskChannelFactory implements TaskChannelFactory { @Override public String getName() { - return "SHELL"; + return NAME; } - @Override - public List getParams() { - List paramsList = new ArrayList<>(); - - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); - - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") - .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) - .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)) - .build(); - - paramsList.add(nodeName); - paramsList.add(runFlag); - return paramsList; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java index 58211ea1ac..b34ab30674 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class SparkTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new SparkTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), SparkParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SparkParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return null; - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java index 92db56ad28..ec6d371b01 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.spark; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class SparkTaskChannelFactory implements TaskChannelFactory { return "SPARK"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new SparkTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java index 79b4098a9c..5b40725669 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java @@ -22,30 +22,18 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class SqlTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new SqlTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), SqlParameters.class); - } - - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, SqlParameters.class).getResources(); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SqlParameters.class); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java index 909741edfc..9627515673 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java @@ -19,23 +19,17 @@ package org.apache.dolphinscheduler.plugin.task.sql; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class SqlTaskChannelFactory implements TaskChannelFactory { - @Override - public String getName() { - return "SQL"; - } + public static final String NAME = "SQL"; @Override - public List getParams() { - return null; + public String getName() { + return NAME; } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java index 098cc3dc01..a22281eca0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannel.java @@ -22,28 +22,18 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; public class SqoopTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new SqoopTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), SqoopParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, SqoopParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, SqoopParameters.class).getResources(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java index 6879af6a6e..f4340c01d5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.sqoop; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class SqoopTaskChannelFactory implements TaskChannelFactory { return "SQOOP"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new SqoopTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java index d9e7318dc4..a784a37498 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java @@ -22,28 +22,17 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public class ZeppelinTaskChannel implements TaskChannel { - @Override - public void cancelApplication(boolean status) { - - } - @Override public AbstractTask createTask(TaskExecutionContext taskRequest) { return new ZeppelinTask(taskRequest); } @Override - public AbstractParameters parseParameters(ParametersNode parametersNode) { - return JSONUtils.parseObject(parametersNode.getTaskParams(), ZeppelinParameters.class); + public AbstractParameters parseParameters(String taskParams) { + return JSONUtils.parseObject(taskParams, ZeppelinParameters.class); } - @Override - public ResourceParametersHelper getResources(String parameters) { - return JSONUtils.parseObject(parameters, ZeppelinParameters.class).getResources(); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java index 006735875c..1afea163bc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java @@ -19,9 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.spi.params.base.PluginParams; - -import java.util.List; import com.google.auto.service.AutoService; @@ -33,11 +30,6 @@ public class ZeppelinTaskChannelFactory implements TaskChannelFactory { return "ZEPPELIN"; } - @Override - public List getParams() { - return null; - } - @Override public TaskChannel create() { return new ZeppelinTaskChannel(); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java index fd430c2b06..a90e4dc6b1 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java @@ -17,10 +17,6 @@ package org.apache.dolphinscheduler.tools.datasource.upgrader.v200; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; - import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.ConditionType; import org.apache.dolphinscheduler.common.enums.Flag; @@ -171,7 +167,7 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { } else { taskDefinitionLog.setResourceIds(""); } - if (TASK_TYPE_SUB_PROCESS.equals(taskType)) { + if ("SUB_PROCESS".equals(taskType)) { JsonNode jsonNodeDefinitionId = param.get("processDefinitionId"); if (jsonNodeDefinitionId != null) { param.put("processDefinitionCode", @@ -197,9 +193,9 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { Constants.FLOWNODE_RUN_FLAG_NORMAL.equals(task.get("runFlag").asText()) ? Flag.YES : Flag.NO); taskDefinitionLog.setTaskType(taskType); taskDefinitionLog.setFailRetryInterval( - TASK_TYPE_SUB_PROCESS.equals(taskType) ? 1 : task.get("retryInterval").asInt()); + "SUB_PROCESS".equals(taskType) ? 1 : task.get("retryInterval").asInt()); taskDefinitionLog.setFailRetryTimes( - TASK_TYPE_SUB_PROCESS.equals(taskType) ? 0 : task.get("maxRetryTimes").asInt()); + "SUB_PROCESS".equals(taskType) ? 0 : task.get("maxRetryTimes").asInt()); taskDefinitionLog.setTaskPriority(JSONUtils .parseObject(JSONUtils.toJsonString(task.get("taskInstancePriority")), Priority.class)); String name = task.get("name").asText(); @@ -241,7 +237,7 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { Map projectIdCodeMap, Map>> processTaskMap) { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { - if (TASK_TYPE_DEPENDENT.equals(taskDefinitionLog.getTaskType())) { + if ("DEPENDENT".equals(taskDefinitionLog.getTaskType())) { ObjectNode taskParams = JSONUtils.parseObject(taskDefinitionLog.getTaskParams()); ObjectNode dependence = (ObjectNode) taskParams.get("dependence"); ArrayNode dependTaskList = @@ -290,7 +286,7 @@ public class V200DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader { private void convertConditions(List taskDefinitionLogList, Map taskNameCodeMap) throws Exception { for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogList) { - if (TASK_TYPE_CONDITIONS.equals(taskDefinitionLog.getTaskType())) { + if ("CONDITIONS".equals(taskDefinitionLog.getTaskType())) { ObjectMapper objectMapper = new ObjectMapper(); ObjectNode taskParams = JSONUtils.parseObject(taskDefinitionLog.getTaskParams()); // reset conditionResult diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 797d8f78a3..a7b49fbd81 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -86,7 +86,7 @@ public class WorkerServer implements IStoppable { @PostConstruct public void run() { this.workerRpcServer.start(); - TaskPluginManager.loadPlugin(); + TaskPluginManager.loadTaskPlugin(); DataSourceProcessorProvider.initialize(); this.workerRegistryClient.setRegistryStoppable(this); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java index c9382f170d..8635ed4600 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java @@ -217,7 +217,7 @@ public abstract class WorkerTaskExecutor implements Runnable { log.info("WorkflowInstanceExecDir: {} check successfully", taskExecutionContext.getExecutePath()); TaskChannel taskChannel = - Optional.ofNullable(TaskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) + Optional.ofNullable(TaskPluginManager.getTaskChannel(taskExecutionContext.getTaskType())) .orElseThrow(() -> new TaskPluginException(taskExecutionContext.getTaskType() + " task plugin not found, please check the task type is correct.")); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index 70db04a6c8..e23e4152cb 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; @@ -69,11 +68,7 @@ public class TaskExecutionContextUtils { public static ResourceContext downloadResourcesIfNeeded(TaskChannel taskChannel, StorageOperator storageOperator, TaskExecutionContext taskExecutionContext) { - AbstractParameters abstractParameters = taskChannel.parseParameters( - ParametersNode.builder() - .taskType(taskExecutionContext.getTaskType()) - .taskParams(taskExecutionContext.getTaskParams()) - .build()); + AbstractParameters abstractParameters = taskChannel.parseParameters(taskExecutionContext.getTaskParams()); List resourceFilesList = abstractParameters.getResourceFilesList(); if (CollectionUtils.isEmpty(resourceFilesList)) {