Browse Source
# Conflicts: # .asf.yaml # dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java # dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java # dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AlertGroupControllerTest.java # dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java # dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java # dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java # dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Command.java # dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java # dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java # dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ScheduleMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.xml # dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/DependentTaskProcessor.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/ConditionsTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DependentTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/MasterBaseTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/MasterTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/SubProcessTaskExecThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/python/PythonTask.java # dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessorTest.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThreadTest.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/PythonCommandExecutorTest.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTaskTest.java # dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTaskTest.java # dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java # dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java # dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java # dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java # dolphinscheduler-standalone-server/pom.xml # dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java # dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java # dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTaskChannelFactory.java # dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonCommandExecutor.java # dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java # dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java # dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java # dolphinscheduler-ui/package.json # dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss # dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue # dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue # sql/dolphinscheduler_h2.sql2.0.7-release
CalvinKirs
3 years ago
9 changed files with 50 additions and 173 deletions
@ -1,155 +0,0 @@ |
|||||||
/* |
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
|
||||||
* contributor license agreements. See the NOTICE file distributed with |
|
||||||
* this work for additional information regarding copyright ownership. |
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
|
||||||
* (the "License"); you may not use this file except in compliance with |
|
||||||
* the License. You may obtain a copy of the License at |
|
||||||
* |
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* |
|
||||||
* Unless required by applicable law or agreed to in writing, software |
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
* See the License for the specific language governing permissions and |
|
||||||
* limitations under the License. |
|
||||||
*/ |
|
||||||
|
|
||||||
package org.apache.dolphinscheduler.server.worker.runner; |
|
||||||
|
|
||||||
import org.apache.dolphinscheduler.common.Constants; |
|
||||||
import org.apache.dolphinscheduler.common.enums.DependResult; |
|
||||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
|
||||||
import org.apache.dolphinscheduler.common.model.DependentItem; |
|
||||||
import org.apache.dolphinscheduler.common.model.DependentTaskModel; |
|
||||||
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; |
|
||||||
import org.apache.dolphinscheduler.common.utils.DependentUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.LoggerUtils; |
|
||||||
import org.apache.dolphinscheduler.common.utils.NetUtils; |
|
||||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
|
||||||
import org.apache.dolphinscheduler.server.utils.LogUtils; |
|
||||||
|
|
||||||
import java.util.ArrayList; |
|
||||||
import java.util.Date; |
|
||||||
import java.util.List; |
|
||||||
import java.util.Map; |
|
||||||
import java.util.concurrent.ConcurrentHashMap; |
|
||||||
|
|
||||||
import org.slf4j.LoggerFactory; |
|
||||||
|
|
||||||
public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { |
|
||||||
|
|
||||||
/** |
|
||||||
* dependent parameters |
|
||||||
*/ |
|
||||||
private DependentParameters dependentParameters; |
|
||||||
|
|
||||||
/** |
|
||||||
* complete task map |
|
||||||
*/ |
|
||||||
private Map<String, ExecutionStatus> completeTaskList = new ConcurrentHashMap<>(); |
|
||||||
|
|
||||||
/** |
|
||||||
* condition result |
|
||||||
*/ |
|
||||||
private DependResult conditionResult; |
|
||||||
|
|
||||||
/** |
|
||||||
* constructor of MasterBaseTaskExecThread |
|
||||||
* |
|
||||||
* @param taskInstance task instance |
|
||||||
*/ |
|
||||||
public ConditionsTaskExecThread(TaskInstance taskInstance) { |
|
||||||
super(taskInstance); |
|
||||||
taskInstance.setStartTime(new Date()); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public Boolean submitWaitComplete() { |
|
||||||
try { |
|
||||||
this.taskInstance = submit(); |
|
||||||
logger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, |
|
||||||
processInstance.getProcessDefinitionCode(), |
|
||||||
processInstance.getProcessDefinitionVersion(), |
|
||||||
taskInstance.getProcessInstanceId(), |
|
||||||
taskInstance.getId())); |
|
||||||
String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, processService.formatTaskAppId(this.taskInstance)); |
|
||||||
Thread.currentThread().setName(threadLoggerInfoName); |
|
||||||
initTaskParameters(); |
|
||||||
logger.info("dependent task start"); |
|
||||||
waitTaskQuit(); |
|
||||||
updateTaskState(); |
|
||||||
} catch (Exception e) { |
|
||||||
logger.error("conditions task run exception", e); |
|
||||||
} |
|
||||||
return true; |
|
||||||
} |
|
||||||
|
|
||||||
private void waitTaskQuit() { |
|
||||||
List<TaskInstance> taskInstances = processService.findValidTaskListByProcessId(taskInstance.getProcessInstanceId()); |
|
||||||
for (TaskInstance task : taskInstances) { |
|
||||||
completeTaskList.putIfAbsent(task.getName(), task.getState()); |
|
||||||
} |
|
||||||
|
|
||||||
List<DependResult> modelResultList = new ArrayList<>(); |
|
||||||
for (DependentTaskModel dependentTaskModel : dependentParameters.getDependTaskList()) { |
|
||||||
List<DependResult> itemDependResult = new ArrayList<>(); |
|
||||||
for (DependentItem item : dependentTaskModel.getDependItemList()) { |
|
||||||
itemDependResult.add(getDependResultForItem(item)); |
|
||||||
} |
|
||||||
DependResult modelResult = DependentUtils.getDependResultForRelation(dependentTaskModel.getRelation(), itemDependResult); |
|
||||||
modelResultList.add(modelResult); |
|
||||||
} |
|
||||||
conditionResult = DependentUtils.getDependResultForRelation(dependentParameters.getRelation(), modelResultList); |
|
||||||
logger.info("the conditions task depend result : {}", conditionResult); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* |
|
||||||
*/ |
|
||||||
private void updateTaskState() { |
|
||||||
ExecutionStatus status; |
|
||||||
if (this.cancel) { |
|
||||||
status = ExecutionStatus.KILL; |
|
||||||
} else { |
|
||||||
status = (conditionResult == DependResult.SUCCESS) ? ExecutionStatus.SUCCESS : ExecutionStatus.FAILURE; |
|
||||||
} |
|
||||||
taskInstance.setState(status); |
|
||||||
taskInstance.setEndTime(new Date()); |
|
||||||
processService.updateTaskInstance(taskInstance); |
|
||||||
} |
|
||||||
|
|
||||||
private void initTaskParameters() { |
|
||||||
taskInstance.setLogPath(LogUtils.getTaskLogPath(processInstance.getProcessDefinitionCode(), |
|
||||||
processInstance.getProcessDefinitionVersion(), |
|
||||||
taskInstance.getProcessInstanceId(), |
|
||||||
taskInstance.getId())); |
|
||||||
this.taskInstance.setHost(NetUtils.getAddr(masterConfig.getListenPort())); |
|
||||||
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); |
|
||||||
taskInstance.setStartTime(new Date()); |
|
||||||
this.processService.saveTaskInstance(taskInstance); |
|
||||||
this.dependentParameters = taskInstance.getDependency(); |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* depend result for depend item |
|
||||||
*/ |
|
||||||
private DependResult getDependResultForItem(DependentItem item) { |
|
||||||
|
|
||||||
DependResult dependResult = DependResult.SUCCESS; |
|
||||||
if (!completeTaskList.containsKey(item.getDepTasks())) { |
|
||||||
logger.info("depend item: {} have not completed yet.", item.getDepTasks()); |
|
||||||
dependResult = DependResult.FAILED; |
|
||||||
return dependResult; |
|
||||||
} |
|
||||||
ExecutionStatus executionStatus = completeTaskList.get(item.getDepTasks()); |
|
||||||
if (executionStatus != item.getStatus()) { |
|
||||||
logger.info("depend item : {} expect status: {}, actual status: {}", item.getDepTasks(), item.getStatus(), executionStatus); |
|
||||||
dependResult = DependResult.FAILED; |
|
||||||
} |
|
||||||
logger.info("dependent item complete {} {},{}", |
|
||||||
Constants.DEPENDENT_SPLIT, item.getDepTasks(), dependResult); |
|
||||||
return dependResult; |
|
||||||
} |
|
||||||
|
|
||||||
} |
|
||||||
@ -0,0 +1,42 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.plugin.task.procedure; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.spi.params.base.PluginParams; |
||||||
|
import org.apache.dolphinscheduler.spi.task.TaskChannel; |
||||||
|
import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; |
||||||
|
|
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class ProcedureTaskChannelFactory implements TaskChannelFactory { |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getName() { |
||||||
|
return "PROCEDURE"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public List<PluginParams> getParams() { |
||||||
|
return null; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public TaskChannel create() { |
||||||
|
return new ProcedureTaskChannel(); |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue