github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Remove dag package in master

Wenjun Ruan 2 months ago
parent
514d713aef
commit
0c2bb5ce25
116 changed files with 321 additions and 4805 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 71
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/BasicDAG.java
  2. 146
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAG.java
  3. 51
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGEdge.java
  4. 148
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGEngine.java
  5. 32
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGEngineFactory.java
  6. 92
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGNode.java
  7. 70
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IDAGEngine.java
  8. 24
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IDAGEngineFactory.java
  9. 35
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IDAGNodeAction.java
  10. 26
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IEventRepositoryFactory.java
  11. 34
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IEventfulExecutionRunnable.java
  12. 24
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionContextFactory.java
  13. 59
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnable.java
  14. 24
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnableFactory.java
  15. 33
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnableRepository.java
  16. 24
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnableRepositoryFactory.java
  17. 25
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowDAG.java
  18. 35
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowDAGFactory.java
  19. 57
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowEngine.java
  20. 46
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionContext.java
  21. 74
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionDAG.java
  22. 23
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionDAGFactory.java
  23. 54
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionRunnable.java
  24. 35
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/MemoryEventRepositoryFactory.java
  25. 35
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionContext.java
  26. 56
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionRunnable.java
  27. 68
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionRunnableRepository.java
  28. 32
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionRunnableRepositoryFactory.java
  29. 27
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionValve.java
  30. 22
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskIdentify.java
  31. 27
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAG.java
  32. 125
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGBuilder.java
  33. 65
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGFactory.java
  34. 82
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowEngine.java
  35. 52
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecuteRunnableRepository.java
  36. 47
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionContext.java
  37. 31
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionContextFactory.java
  38. 116
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionDAG.java
  39. 60
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionDAGFactory.java
  40. 64
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnable.java
  41. 40
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableFactory.java
  42. 35
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowIdentify.java
  43. 111
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/EventEngine.java
  44. 95
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/EventFirer.java
  45. 36
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/EventOperatorManager.java
  46. 24
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IAsyncEvent.java
  47. 22
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEvent.java
  48. 32
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventOperator.java
  49. 33
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventOperatorManager.java
  50. 37
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventRepository.java
  51. 21
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/ISyncEvent.java
  52. 26
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/ITaskEvent.java
  53. 22
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/ITaskEventOperator.java
  54. 29
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IWorkflowEvent.java
  55. 23
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IWorkflowEventOperator.java
  56. 60
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/MemoryEventRepository.java
  57. 36
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskLogSendToRemoteEvent.java
  58. 38
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskLogSendToRemoteEventOperator.java
  59. 46
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskOperationEvent.java
  60. 47
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskOperationEventOperator.java
  61. 29
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskOperationType.java
  62. 33
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskSuccessEvent.java
  63. 47
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskSuccessEventOperator.java
  64. 35
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFailedEvent.java
  65. 57
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFailedEventOperator.java
  66. 32
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFinalizeEvent.java
  67. 42
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFinalizeEventOperator.java
  68. 37
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFinishEvent.java
  69. 51
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationEvent.java
  70. 87
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationEventOperator.java
  71. 33
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTimeoutEvent.java
  72. 61
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTimeoutEventOperator.java
  73. 38
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggerNextTaskEvent.java
  74. 44
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggerNextTaskEventOperator.java
  75. 33
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggeredEvent.java
  76. 63
      dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggeredEventOperator.java
  77. 121
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/DAGNodeTest.java
  78. 88
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/MockWorkflowExecutionContextFactory.java
  79. 31
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/MockWorkflowExecutionRunnableFactory.java
  80. 171
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGAssertion.java
  81. 282
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGBuilderTest.java
  82. 114
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowEngineTest.java
  83. 48
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableAssertions.java
  84. 43
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableRepositoryAssertions.java
  85. 51
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableTest.java
  86. 46
      dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationEventTest.java
  87. 66
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/EventEngine.java
  88. 72
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/EventEngineFactory.java
  89. 16
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/EventFirer.java
  90. 8
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventAcceptor.java
  91. 8
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventDispatcher.java
  92. 18
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventEngine.java
  93. 6
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventEngineFactory.java
  94. 13
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventFirer.java
  95. 13
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IWorkflowEngine.java
  96. 21
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/WorkflowEngine.java
  97. 30
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/WorkflowEngineFactory.java
  98. 29
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/EventOperatorManager.java
  99. 2
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/IEvent.java
  100. 35
      dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/IEventFirer.java
  101. Some files were not shown because too many files have changed in this diff Show More

71
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/BasicDAG.java
Unescape View File

@ -1,71 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@SuperBuilder
public abstract class BasicDAG implements DAG {
protected Map<String, DAGNode> dagNodeMap;
@Override
public List<DAGNode> getDirectPostNodes(DAGNode dagNode) {
final String nodeName = dagNode.getNodeName();
if (!dagNodeMap.containsKey(nodeName)) {
return Collections.emptyList();
}
DAGNode node = dagNodeMap.get(nodeName);
List<DAGNode> dagNodes = new ArrayList<>();
for (DAGEdge edge : node.getOutDegrees()) {
if (dagNodeMap.containsKey(edge.getToNodeName())) {
dagNodes.add(dagNodeMap.get(edge.getToNodeName()));
}
}
return dagNodes;
}
@Override
public List<DAGNode> getDirectPreNodes(DAGNode dagNode) {
final String nodeName = dagNode.getNodeName();
if (!dagNodeMap.containsKey(nodeName)) {
return Collections.emptyList();
}
DAGNode node = dagNodeMap.get(nodeName);
List<DAGNode> dagNodes = new ArrayList<>();
for (DAGEdge edge : node.getInDegrees()) {
if (dagNodeMap.containsKey(edge.getFromNodeName())) {
dagNodes.add(dagNodeMap.get(edge.getFromNodeName()));
}
}
return dagNodes;
}
@Override
public DAGNode getDAGNode(String nodeName) {
return dagNodeMap.get(nodeName);
}
}

146
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAG.java
Unescape View File

@ -1,146 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.List;
import java.util.stream.Collectors;
/**
* The Directed Acyclic Graph class.
* <p>
* The DAG is a directed graph, which contains the nodes and the edges, the nodeName is the unique identifier of the node.
* The nodes are the tasks, the edges are the dependencies between the tasks.
* The DAG is acyclic, which means there is no cycle in the graph.
* The DAG is a directed graph, which means the edges have direction.
*
*/
public interface DAG {
/**
* Get the direct post node of given dagNode, if the dagNode is null, return the nodes which doesn't have inDegrees.
* e.g. The DAG is:
* <pre>
* {@code
* 1 -> 2 -> 3
* 4 -> 5
* 6
* }
* </pre>
* <li> The post node of 1 is 2.
* <li> The post node of 3 is empty.
* <li> The post node of null is 1,4,6.
*
* @param dagNode the node of the DAG, can be null.
* @return post node list, sort by priority.
*/
List<DAGNode> getDirectPostNodes(DAGNode dagNode);
/**
* Same with {@link #getDirectPostNodes(DAGNode)}.
* <p>
* If the dagNodeName is null, return the nodes which doesn't have inDegrees. Otherwise, return the post nodes of
* the given dagNodeName. If the dagNodeName is not null and cannot find the node in DAG, throw IllegalArgumentException.
*
* @param dagNodeName task name, can be null.
* @return post task name list, sort by priority.
* @throws IllegalArgumentException if the dagNodeName is not null and cannot find the node in DAG.
*/
default List<DAGNode> getDirectPostNodes(String dagNodeName) {
DAGNode dagNode = getDAGNode(dagNodeName);
if (dagNodeName != null && dagNode == null) {
throw new IllegalArgumentException("Cannot find the Node: " + dagNodeName + " in DAG");
}
return getDirectPostNodes(dagNode);
}
/**
* Same with {@link #getDirectPostNodes(String)}. Return the post node names.
*
* @param dagNodeName task name, can be null.
* @return post task name list, sort by priority.
* @throws IllegalArgumentException if the dagNodeName is not null and cannot find the node in DAG.
*/
default List<String> getDirectPostNodeNames(String dagNodeName) {
DAGNode dagNode = getDAGNode(dagNodeName);
if (dagNodeName != null && dagNode == null) {
throw new IllegalArgumentException("Cannot find the Node: " + dagNodeName + " in DAG");
}
return getDirectPostNodes(dagNode).stream().map(DAGNode::getNodeName).collect(Collectors.toList());
}
/**
* Get the direct pre node of given dagNode, if the dagNode is null, return the nodes which doesn't have outDegrees.
* e.g. The DAG is:
* <pre>
* {@code
* 1 -> 2 -> 3
* 4 -> 5
* 6
* }
* </pre>
* <li> The pre node of 1 is empty.
* <li> The pre node of 3 is 2.
* <li> The pre node of null is 3,5,6.
*
* @param dagNode the node of the DAG, can be null.
* @return pre node list, sort by priority.
*/
List<DAGNode> getDirectPreNodes(DAGNode dagNode);
/**
* Same with {@link #getDirectPreNodes(DAGNode)}.
* <p>
* If the dagNodeName is null, return the nodes which doesn't have outDegrees. Otherwise, return the pre nodes of
* the given dagNodeName. If the dagNodeName is not null and cannot find the node in DAG, throw IllegalArgumentException.
*
* @param dagNodeName task name, can be null.
* @return pre task name list, sort by priority.
* @throws IllegalArgumentException if the dagNodeName is not null and cannot find the node in DAG.
*/
default List<DAGNode> getDirectPreNodes(String dagNodeName) {
DAGNode dagNode = getDAGNode(dagNodeName);
if (dagNodeName != null && dagNode == null) {
throw new IllegalArgumentException("Cannot find the Node: " + dagNodeName + " in DAG");
}
return getDirectPreNodes(dagNode);
}
/**
* Same with {@link #getDirectPreNodes(String)}. Return the pre node names.
*
* @param dagNodeName task name, can be null.
* @return pre task name list, sort by priority.
* @throws IllegalArgumentException if the dagNodeName is not null and cannot find the node in DAG.
*/
default List<String> getDirectPreNodeNames(String dagNodeName) {
DAGNode dagNode = getDAGNode(dagNodeName);
if (dagNodeName != null && dagNode == null) {
throw new IllegalArgumentException("Cannot find the Node: " + dagNodeName + " in DAG");
}
return getDirectPreNodes(dagNode).stream().map(DAGNode::getNodeName).collect(Collectors.toList());
}
/**
* Get the node of the DAG by the node name.
*
* @param nodeName the node name.
* @return the node of the DAG, return null if cannot find the node.
*/
DAGNode getDAGNode(String nodeName);
}

51
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGEdge.java
Unescape View File

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.Builder;
import lombok.Data;
/**
* The edge of the DAG.
* <p>
* The edge contains the fromNodeName and the toNodeName, the fromNodeName is the node name of the from node, the toNodeName is the node name of the to node.
* <p>
* The formNodeName can be null, which means the edge is from the start node of the DAG.
* The toNodeName can be null, which means the edge is to the end node of the DAG.
* The fromNodeName and the toNodeName cannot be null at the same time.
*/
@Data
@Builder
public class DAGEdge {
private String fromNodeName;
private String toNodeName;
public DAGEdge(String fromNodeName, String toNodeName) {
if (fromNodeName == null && toNodeName == null) {
throw new IllegalArgumentException("fromNodeName and toNodeName cannot be null at the same time"
+ "fromNodeName: " + fromNodeName + ", toNodeName: " + toNodeName);
}
if (fromNodeName != null && fromNodeName.equals(toNodeName)) {
throw new IllegalArgumentException("fromNodeName and toNodeName cannot be the same"
+ "fromNodeName: " + fromNodeName + ", toNodeName: " + toNodeName);
}
this.fromNodeName = fromNodeName;
this.toNodeName = toNodeName;
}
}

148
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGEngine.java
Unescape View File

@ -1,148 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.TaskOperationEvent;
import org.apache.dolphinscheduler.server.master.events.TaskOperationType;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DAGEngine implements IDAGEngine {
private final IWorkflowExecutionContext workflowExecutionContext;
public DAGEngine(IWorkflowExecutionContext workflowExecutionContext) {
this.workflowExecutionContext = workflowExecutionContext;
}
@Override
public void triggerNextTasks(String parentTaskNodeName) {
workflowExecutionContext.getWorkflowExecutionDAG()
.getDirectPostNodeNames(parentTaskNodeName)
.forEach(this::triggerTask);
}
@Override
public void triggerTask(String taskName) {
IWorkflowExecutionDAG workflowExecutionDAG = workflowExecutionContext.getWorkflowExecutionDAG();
DAGNode dagNode = workflowExecutionDAG.getDAGNode(taskName);
if (dagNode == null) {
log.error("Cannot find the DAGNode for task: {}", taskName);
return;
}
// Use condition check?
if (!workflowExecutionDAG.isTaskAbleToBeTriggered(taskName)) {
log.info("The task: {} is not able to be triggered", taskName);
return;
}
if (dagNode.isSkip()) {
log.info("The task: {} is skipped", taskName);
triggerNextTasks(taskName);
return;
}
TaskExecutionRunnable taskExecutionRunnable = workflowExecutionDAG.triggerTask(taskName);
TaskOperationEvent taskOperationEvent = TaskOperationEvent.builder()
.taskExecutionRunnable(taskExecutionRunnable)
.taskOperationType(TaskOperationType.RUN)
.build();
workflowExecutionContext.getEventRepository().storeEventToTail(taskOperationEvent);
}
@Override
public void failoverTask(Integer taskInstanceId) {
TaskExecutionRunnable taskExecutionRunnable =
workflowExecutionContext.getWorkflowExecutionDAG().getTaskExecutionRunnableById(taskInstanceId);
if (taskExecutionRunnable == null) {
log.error("Cannot find the ITaskExecutionRunnable for taskInstance: {}", taskInstanceId);
return;
}
TaskOperationEvent taskOperationEvent = TaskOperationEvent.builder()
.taskExecutionRunnable(taskExecutionRunnable)
.taskOperationType(TaskOperationType.FAILOVER)
.build();
workflowExecutionContext.getEventRepository().storeEventToTail(taskOperationEvent);
}
@Override
public void retryTask(Integer taskInstanceId) {
TaskExecutionRunnable taskExecutionRunnable =
workflowExecutionContext.getWorkflowExecutionDAG().getTaskExecutionRunnableById(taskInstanceId);
if (taskExecutionRunnable == null) {
log.error("Cannot find the ITaskExecutionRunnable for taskInstance: {}", taskInstanceId);
return;
}
TaskOperationEvent taskOperationEvent = TaskOperationEvent.builder()
.taskExecutionRunnable(taskExecutionRunnable)
.taskOperationType(TaskOperationType.RETRY)
.build();
workflowExecutionContext.getEventRepository().storeEventToTail(taskOperationEvent);
}
@Override
public void pauseAllTask() {
workflowExecutionContext.getWorkflowExecutionDAG()
.getActiveTaskExecutionRunnable()
.stream()
.map(taskExecutionRunnable -> taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getId())
.forEach(this::pauseTask);
}
@Override
public void pauseTask(Integer taskInstanceId) {
TaskExecutionRunnable taskExecutionRunnable =
workflowExecutionContext.getWorkflowExecutionDAG().getTaskExecutionRunnableById(taskInstanceId);
if (taskExecutionRunnable == null) {
log.error("Cannot find the ITaskExecutionRunnable for taskInstance: {}", taskInstanceId);
return;
}
TaskOperationEvent taskOperationEvent = TaskOperationEvent.builder()
.taskExecutionRunnable(taskExecutionRunnable)
.taskOperationType(TaskOperationType.PAUSE)
.build();
workflowExecutionContext.getEventRepository().storeEventToTail(taskOperationEvent);
}
@Override
public void killAllTask() {
workflowExecutionContext.getWorkflowExecutionDAG()
.getActiveTaskExecutionRunnable()
.stream()
.map(taskExecutionRunnable -> taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getId())
.forEach(this::killTask);
}
@Override
public void killTask(Integer taskInstanceId) {
TaskExecutionRunnable taskExecutionRunnable =
workflowExecutionContext.getWorkflowExecutionDAG().getTaskExecutionRunnableById(taskInstanceId);
if (taskExecutionRunnable == null) {
log.error("Cannot find the ITaskExecutionRunnable for taskInstance: {}", taskInstanceId);
return;
}
TaskOperationEvent taskOperationEvent = TaskOperationEvent.builder()
.taskExecutionRunnable(taskExecutionRunnable)
.taskOperationType(TaskOperationType.KILL)
.build();
workflowExecutionContext.getEventRepository().storeEventToTail(taskOperationEvent);
}
}

32
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGEngineFactory.java
Unescape View File

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class DAGEngineFactory implements IDAGEngineFactory {
@Override
public IDAGEngine createDAGEngine(IWorkflowExecutionContext workflowExecutionContext) {
return new DAGEngine(workflowExecutionContext);
}
}

92
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/DAGNode.java
Unescape View File

@ -1,92 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
/**
* The node of the DAG.
* <p>
* The node contains the node name, the content of the node, the inDegrees and the outDegrees.
* The inDegrees is the edge from other nodes to the current node, the outDegrees is the edge from the current
* node to other nodes.
*/
@Getter
@Builder
@NoArgsConstructor
public class DAGNode {
private String nodeName;
/**
* whether the node is skipped, default is false, which means the node is not skipped.
* If the node is skipped, the node will not be executed.
*/
@Builder.Default
private boolean skip = false;
private List<DAGEdge> inDegrees;
private List<DAGEdge> outDegrees;
public DAGNode(String nodeName,
List<DAGEdge> inDegrees,
List<DAGEdge> outDegrees) {
this(nodeName, false, inDegrees, outDegrees);
}
public DAGNode(String nodeName,
boolean skip,
List<DAGEdge> inDegrees,
List<DAGEdge> outDegrees) {
if (StringUtils.isEmpty(nodeName)) {
throw new IllegalArgumentException("nodeName cannot be empty");
}
if (CollectionUtils.isNotEmpty(inDegrees)) {
inDegrees.forEach(dagEdge -> {
if (!nodeName.equals(dagEdge.getToNodeName())) {
throw new IllegalArgumentException(
"The toNodeName of inDegree should be the nodeName of the node: "
+ nodeName + ", inDegree: " + dagEdge);
}
});
}
if (CollectionUtils.isNotEmpty(outDegrees)) {
outDegrees.forEach(dagEdge -> {
if (!nodeName.equals(dagEdge.getFromNodeName())) {
throw new IllegalArgumentException(
"The fromNodeName of outDegree should be the nodeName of the node: "
+ nodeName + ", outDegree: " + dagEdge);
}
});
}
this.nodeName = nodeName;
this.inDegrees = inDegrees;
this.outDegrees = outDegrees;
this.skip = skip;
}
}

70
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IDAGEngine.java
Unescape View File

@ -1,70 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
/**
* The IDAGEngine is responsible for triggering, killing, pausing, and finalizing task in {@link IWorkflowExecutionDAG}.
* <p>All DAG operation should directly use the method in IDAGEngine, new {@link IWorkflowExecutionDAG} should be triggered by new IDAGEngine.
*/
public interface IDAGEngine {
/**
* Trigger the tasks which are post of the given task.
* <P> If there are no task after the given taskNode, will try to finish the WorkflowExecutionRunnable.
* <p> If the
*
* @param parentTaskNodeName the parent task name
*/
void triggerNextTasks(String parentTaskNodeName);
/**
* Trigger the given task
*
* @param taskName task name
*/
void triggerTask(String taskName);
/**
* Failover the given task.
*
* @param taskInstanceId taskInstanceId
*/
void failoverTask(Integer taskInstanceId);
/**
* Retry the given task.
*
* @param taskInstanceId taskInstanceId
*/
void retryTask(Integer taskInstanceId);
void pauseAllTask();
/**
* Pause the given task.
*/
void pauseTask(Integer taskInstanceId);
void killAllTask();
/**
* Kill the given task.
*/
void killTask(Integer taskId);
}

24
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IDAGEngineFactory.java
Unescape View File

@ -1,24 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
public interface IDAGEngineFactory {
IDAGEngine createDAGEngine(IWorkflowExecutionContext workflowExecutionContext);
}

35
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IDAGNodeAction.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
/**
* The IDAGNodeAction represent the action of a DAG node.
*/
public interface IDAGNodeAction {
void run();
void kill();
void pause();
void success();
void failure();
}

26
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IEventRepositoryFactory.java
Unescape View File

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.IEventRepository;
public interface IEventRepositoryFactory {
IEventRepository createEventRepository();
}

34
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IEventfulExecutionRunnable.java
Unescape View File

@ -1,34 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.IEvent;
import org.apache.dolphinscheduler.server.master.events.IEventRepository;
public interface IEventfulExecutionRunnable {
IEventRepository getEventRepository();
default void storeEventToTail(IEvent event) {
getEventRepository().storeEventToTail(event);
}
default void storeEventToHead(IEvent event) {
getEventRepository().storeEventToHead(event);
}
}

24
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionContextFactory.java
Unescape View File

@ -1,24 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
public interface ITaskExecutionContextFactory {
ITaskExecutionContext createTaskExecutionContext();
}

59
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnable.java
Unescape View File

@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
/**
* The TaskExecutionRunnable represent the running task, it is responsible for operate the task instance. e.g. dispatch, kill, pause.
*/
public interface ITaskExecutionRunnable {
/**
* Dispatch the task instance.
*/
void dispatch();
/**
* Run the task instance.
*/
void run();
/**
* Kill the task instance.
*/
void kill();
/**
* Pause the task instance.
*/
void pause();
/**
* Get the task execution context.
*
* @return the task execution context
*/
TaskExecutionContext getTaskExecutionContext();
/**
* Determine whether the current task can be accessed to the post task.
*
* @param taskNodeName post task name
* @return true if the current task can be accessed to the post task.
*/
boolean canAccessTo(String taskNodeName);
}

24
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnableFactory.java
Unescape View File

@ -1,24 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
public interface ITaskExecutionRunnableFactory {
TaskExecutionRunnable createTaskExecutionRunnable(ITaskExecutionContext taskExecutionContext);
}

33
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnableRepository.java
Unescape View File

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.Collection;
public interface ITaskExecutionRunnableRepository {
void storeTaskExecutionRunnable(TaskExecutionRunnable taskExecutionRunnable);
TaskExecutionRunnable getTaskExecutionRunnableById(Integer taskInstanceId);
TaskExecutionRunnable getTaskExecutionRunnableByName(String taskInstanceName);
Collection<TaskExecutionRunnable> getActiveTaskExecutionRunnable();
void removeTaskExecutionRunnable(Integer taskInstanceId);
}

24
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionRunnableRepositoryFactory.java
Unescape View File

@ -1,24 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
public interface ITaskExecutionRunnableRepositoryFactory {
TaskExecutionRunnableRepository createTaskExecutionRunnableRepository();
}

25
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowDAG.java
Unescape View File

@ -1,25 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
/**
* The IWorkflowDAG represent the DAG of a workflow.
*/
public interface IWorkflowDAG extends DAG {
}

35
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowDAGFactory.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
/**
* The Factory used to create {@link IWorkflowDAG}
*/
public interface IWorkflowDAGFactory {
/**
* Create the WorkflowDAG
*
* @param workflowInstance workflowInstance.
* @return workflow DAG.
*/
IWorkflowDAG createWorkflowDAG(ProcessInstance workflowInstance);
}

57
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowEngine.java
Unescape View File

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.exception.WorkflowExecuteRunnableNotFoundException;
/**
* The WorkflowEngine is responsible for starting, stopping, pausing, and finalizing workflows.
*/
public interface IWorkflowEngine {
/**
* Trigger a workflow to start.
*
* @param workflowExecuteRunnable the workflow to start
*/
void triggerWorkflow(IWorkflowExecutionRunnable workflowExecuteRunnable);
/**
* Pause a workflow instance.
*
* @param workflowInstanceId the ID of the workflow to pause
* @throws WorkflowExecuteRunnableNotFoundException if the workflow is not found
*/
void pauseWorkflow(Integer workflowInstanceId);
/**
* Kill a workflow instance.
*
* @param workflowInstanceId the ID of the workflow to stop
* @throws WorkflowExecuteRunnableNotFoundException if the workflow is not found
*/
void killWorkflow(Integer workflowInstanceId);
/**
* Finalize a workflow instance. Once a workflow has been finalized, then it cannot receive new operation, and will be removed from memory.
*
* @param workflowInstanceId the ID of the workflow to finalize
*/
void finalizeWorkflow(Integer workflowInstanceId);
}

46
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionContext.java
Unescape View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.master.events.IEventRepository;
import java.util.List;
public interface IWorkflowExecutionContext {
IWorkflowExecutionDAG getWorkflowExecutionDAG();
ProcessInstance getWorkflowInstance();
ProcessDefinition getWorkflowDefinition();
List<String> getBeginNodeNames();
IEventRepository getEventRepository();
default int getWorkflowInstanceId() {
return getWorkflowInstance().getId();
}
default String getWorkflowInstanceName() {
return getWorkflowInstance().getName();
}
}

74
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionDAG.java
Unescape View File

@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.List;
/**
* The WorkflowExecutionDAG represent the running workflow DAG.
*/
public interface IWorkflowExecutionDAG extends DAG {
/**
* Trigger the taskNode by given taskName.
*
* @param taskName taskNodeName
* @return TaskExecutionRunnable
*/
TaskExecutionRunnable triggerTask(String taskName);
/**
* Get TaskExecutionRunnable by given TaskInstanceId.
*
* @param taskInstanceId taskInstanceId.
* @return TaskExecutionRunnable
*/
TaskExecutionRunnable getTaskExecutionRunnableById(Integer taskInstanceId);
/**
* Get TaskExecutionRunnable by given taskName.
*
* @param taskName task name.
* @return TaskExecutionRunnable
*/
TaskExecutionRunnable getTaskExecutionRunnableByName(String taskName);
/**
* Get TaskExecutionRunnable which is not finished.
*
* @return TaskExecutionRunnable
*/
List<TaskExecutionRunnable> getActiveTaskExecutionRunnable();
/**
* Get the direct pre TaskExecutionRunnable of the given taskName.
*
* @param taskName task name.
* @return TaskExecutionRunnable
*/
List<TaskExecutionRunnable> getDirectPreTaskExecutionRunnable(String taskName);
/**
* Check whether the taskNode is ready to run.
*
* @param taskName taskNodeName
* @return true if the taskNode is ready to run.
*/
boolean isTaskAbleToBeTriggered(String taskName);
}

23
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionDAGFactory.java
Unescape View File

@ -1,23 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
public interface IWorkflowExecutionDAGFactory {
IWorkflowExecutionDAG createWorkflowExecutionDAG();
}

54
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/IWorkflowExecutionRunnable.java
Unescape View File

@ -1,54 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
/**
* The IWorkflowExecuteRunnable represent a running workflow instance, it is responsible for operate the workflow instance. e.g. start, kill, pause.
*/
public interface IWorkflowExecutionRunnable extends IEventfulExecutionRunnable {
/**
* Start the workflow instance.
*/
void start();
/**
* Kill the workflow instance.
*/
void kill();
/**
* Pause the workflow instance.
*/
void pause();
/**
* Get the workflow execution context.
*
* @return the workflow execution context
*/
IWorkflowExecutionContext getWorkflowExecutionContext();
/**
* Get the {@link IDAGEngine} which used to execute the dag of the workflow instance.
*
* @return dag engine.
*/
IDAGEngine getDagEngine();
}

35
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/MemoryEventRepositoryFactory.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.IEventRepository;
import org.apache.dolphinscheduler.server.master.events.MemoryEventRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class MemoryEventRepositoryFactory implements IEventRepositoryFactory {
@Override
public IEventRepository createEventRepository() {
return new MemoryEventRepository();
}
}

35
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionContext.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class TaskExecutionContext implements ITaskExecutionContext {
private TaskInstance taskInstance;
}

56
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionRunnable.java
Unescape View File

@ -1,56 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Getter
@Builder
public class TaskExecutionRunnable implements ITaskExecutionRunnable {
private TaskExecutionContext taskExecutionContext;
@Override
public void dispatch() {
// todo: check if the operation is valid
}
@Override
public void run() {
}
@Override
public void kill() {
// todo: check if the operation is valid
}
@Override
public void pause() {
// todo: check if the operation is valid
}
@Override
public boolean canAccessTo(String postTaskNodeName) {
return false;
}
}

68
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionRunnableRepository.java
Unescape View File

@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* Use to store the TaskExecutionRunnable of a DAG.
*/
public class TaskExecutionRunnableRepository implements ITaskExecutionRunnableRepository {
private final Map<Integer, TaskExecutionRunnable> taskExecuteRunnableMap = new ConcurrentHashMap<>();
private final Map<String, TaskExecutionRunnable> taskExecuteRunnableNameMap = new ConcurrentHashMap<>();
@Override
public void storeTaskExecutionRunnable(TaskExecutionRunnable taskExecutionRunnable) {
taskExecuteRunnableMap.put(taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getId(),
taskExecutionRunnable);
taskExecuteRunnableNameMap.put(taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getName(),
taskExecutionRunnable);
}
public TaskExecutionRunnable getTaskExecutionRunnableById(Integer taskInstanceId) {
return taskExecuteRunnableMap.get(taskInstanceId);
}
@Override
public TaskExecutionRunnable getTaskExecutionRunnableByName(String taskInstanceName) {
return taskExecuteRunnableNameMap.get(taskInstanceName);
}
public Collection<TaskExecutionRunnable> getActiveTaskExecutionRunnable() {
return taskExecuteRunnableMap.values()
.stream()
.filter(taskExecutionRunnable -> {
return taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getState().isRunning();
})
.collect(Collectors.toList());
}
public void removeTaskExecutionRunnable(Integer taskInstanceId) {
TaskExecutionRunnable taskExecutionRunnable = taskExecuteRunnableMap.remove(taskInstanceId);
if (taskExecutionRunnable != null) {
taskExecuteRunnableNameMap
.remove(taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getName());
}
}
}

32
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionRunnableRepositoryFactory.java
Unescape View File

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class TaskExecutionRunnableRepositoryFactory implements ITaskExecutionRunnableRepositoryFactory {
public TaskExecutionRunnableRepository createTaskExecutionRunnableRepository() {
return new TaskExecutionRunnableRepository();
}
}

27
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskExecutionValve.java
Unescape View File

@ -1,27 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
/**
* Used to judge whether the task can be executed.
*/
public interface TaskExecutionValve {
boolean canOpen(TaskExecutionRunnable preTask, String postTask);
}

22
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/TaskIdentify.java
Unescape View File

@ -1,22 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
public class TaskIdentify {
}

27
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAG.java
Unescape View File

@ -1,27 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@SuperBuilder
public class WorkflowDAG extends BasicDAG implements IWorkflowDAG {
}

125
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGBuilder.java
Unescape View File

@ -1,125 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Used to build WorkflowDAG, you need to add TaskNode first, then add TaskEdge.
* After adding all the TaskNodes and TaskEdges, you can call the build method to get the WorkflowDAG.
* <p>
* Example:
* <pre>
* {@code
* WorkflowDAG workflowDAG = WorkflowDAGBuilder.newBuilder()
* .addTaskNode(taskNodeA)
* .addTaskNode(taskNodeB)
* .addTaskNode(taskNodeC)
* .addTaskEdge(edgeAB)
* .addTaskEdge(edgeBC)
* .build();
* }
* </pre>
*/
public class WorkflowDAGBuilder {
private final Map<String, DAGNode> taskNameMap;
private final Map<Long, DAGNode> taskCodeMap;
private WorkflowDAGBuilder() {
this.taskCodeMap = new HashMap<>();
this.taskNameMap = new HashMap<>();
}
public static WorkflowDAGBuilder newBuilder() {
return new WorkflowDAGBuilder();
}
public WorkflowDAGBuilder addTaskNodes(List<TaskDefinitionLog> taskDefinitionList) {
taskDefinitionList.forEach(this::addTaskNode);
return this;
}
public WorkflowDAGBuilder addTaskNode(TaskDefinitionLog taskDefinition) {
String taskName = taskDefinition.getName();
long taskCode = taskDefinition.getCode();
if (taskCodeMap.containsKey(taskCode)) {
throw new IllegalArgumentException("TaskNode with code " + taskCode + " already exists");
}
if (taskNameMap.containsKey(taskName)) {
throw new IllegalArgumentException("TaskNode with name " + taskName + " already exists");
}
DAGNode taskNode = DAGNode.builder()
.nodeName(taskName)
.inDegrees(new ArrayList<>())
.outDegrees(new ArrayList<>())
.skip(Flag.NO.equals(taskDefinition.getFlag()))
.build();
taskNameMap.put(taskName, taskNode);
taskCodeMap.put(taskCode, taskNode);
return this;
}
public WorkflowDAGBuilder addTaskEdges(List<ProcessTaskRelationLog> processTaskRelations) {
processTaskRelations.forEach(this::addTaskEdge);
return this;
}
public WorkflowDAGBuilder addTaskEdge(ProcessTaskRelationLog processTaskRelation) {
long preTaskCode = processTaskRelation.getPreTaskCode();
long postTaskCode = processTaskRelation.getPostTaskCode();
if (taskCodeMap.containsKey(preTaskCode)) {
DAGNode fromTask = taskCodeMap.get(preTaskCode);
if (taskCodeMap.containsKey(postTaskCode)) {
DAGNode toTask = taskCodeMap.get(postTaskCode);
DAGEdge edge = DAGEdge.builder()
.fromNodeName(fromTask.getNodeName())
.toNodeName(toTask.getNodeName())
.build();
if (fromTask.getOutDegrees().contains(edge)) {
throw new IllegalArgumentException(
"Edge from " + fromTask.getNodeName() + " to " + toTask.getNodeName() + " already exists");
}
fromTask.getOutDegrees().add(edge);
if (toTask.getInDegrees().contains(edge)) {
throw new IllegalArgumentException(
"Edge from " + fromTask.getNodeName() + " to " + toTask.getNodeName() + " already exists");
}
toTask.getInDegrees().add(edge);
}
}
return this;
}
public WorkflowDAG build() {
return WorkflowDAG.builder()
.dagNodeMap(taskNameMap)
.build();
}
}

65
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGFactory.java
Unescape View File

@ -1,65 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowDAGFactory implements IWorkflowDAGFactory {
@Autowired
private TaskDefinitionLogDao taskDefinitionLogDao;
@Autowired
private ProcessTaskRelationLogDao processTaskRelationLogDao;
@Override
public IWorkflowDAG createWorkflowDAG(ProcessInstance processInstance) {
Long workflowDefinitionCode = processInstance.getProcessDefinitionCode();
Integer workflowDefinitionVersion = processInstance.getProcessDefinitionVersion();
List<TaskDefinitionLog> taskDefinitions = queryTaskNodes(workflowDefinitionCode, workflowDefinitionVersion);
List<ProcessTaskRelationLog> taskRelations = queryTaskEdges(workflowDefinitionCode, workflowDefinitionVersion);
return WorkflowDAGBuilder.newBuilder()
.addTaskNodes(taskDefinitions)
.addTaskEdges(taskRelations)
.build();
}
private List<TaskDefinitionLog> queryTaskNodes(Long workflowDefinitionCode, Integer workflowDefinitionVersion) {
return taskDefinitionLogDao.queryByWorkflowDefinitionCodeAndVersion(workflowDefinitionCode,
workflowDefinitionVersion);
}
private List<ProcessTaskRelationLog> queryTaskEdges(Long workflowDefinitionCode,
Integer workflowDefinitionVersion) {
return processTaskRelationLogDao.queryByWorkflowDefinitionCodeAndVersion(workflowDefinitionCode,
workflowDefinitionVersion);
}
}

82
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowEngine.java
Unescape View File

@ -1,82 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.WorkflowOperationEvent;
import org.apache.dolphinscheduler.server.master.exception.WorkflowExecuteRunnableNotFoundException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowEngine implements IWorkflowEngine {
private final WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
public WorkflowEngine(WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository) {
this.workflowExecuteRunnableRepository = workflowExecuteRunnableRepository;
}
@Override
public void triggerWorkflow(IWorkflowExecutionRunnable workflowExecuteRunnable) {
IWorkflowExecutionContext workflowExecutionContext = workflowExecuteRunnable.getWorkflowExecutionContext();
Integer workflowInstanceId = workflowExecutionContext.getWorkflowInstanceId();
log.info("Triggering WorkflowExecutionRunnable: {}", workflowExecutionContext.getWorkflowInstanceName());
workflowExecuteRunnableRepository.storeWorkflowExecutionRunnable(workflowExecuteRunnable);
workflowExecuteRunnable.storeEventToTail(WorkflowOperationEvent.triggerEvent(workflowInstanceId));
}
@Override
public void pauseWorkflow(Integer workflowInstanceId) {
IWorkflowExecutionRunnable workflowExecuteRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId);
if (workflowExecuteRunnable == null) {
throw new WorkflowExecuteRunnableNotFoundException(workflowInstanceId);
}
log.info("Pausing WorkflowExecutionRunnable: {}",
workflowExecuteRunnable.getWorkflowExecutionContext().getWorkflowInstanceName());
workflowExecuteRunnable.storeEventToTail(WorkflowOperationEvent.pauseEvent(workflowInstanceId));
}
@Override
public void killWorkflow(Integer workflowInstanceId) {
IWorkflowExecutionRunnable workflowExecuteRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId);
if (workflowExecuteRunnable == null) {
throw new WorkflowExecuteRunnableNotFoundException(workflowInstanceId);
}
log.info("Killing WorkflowExecutionRunnable: {}",
workflowExecuteRunnable.getWorkflowExecutionContext().getWorkflowInstanceName());
workflowExecuteRunnable.storeEventToTail(WorkflowOperationEvent.killEvent(workflowInstanceId));
}
@Override
public void finalizeWorkflow(Integer workflowInstanceId) {
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId);
if (workflowExecutionRunnable == null) {
return;
}
log.info("Finalizing WorkflowExecutionRunnable: {}",
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstanceName());
workflowExecuteRunnableRepository.removeWorkflowExecutionRunnable(workflowInstanceId);
}
}

52
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecuteRunnableRepository.java
Unescape View File

@ -1,52 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowExecuteRunnableRepository {
private final Map<Integer, IWorkflowExecutionRunnable> workflowExecutionRunnableMap = new ConcurrentHashMap<>();
public void storeWorkflowExecutionRunnable(IWorkflowExecutionRunnable workflowExecutionRunnable) {
workflowExecutionRunnableMap.put(
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstanceId(),
workflowExecutionRunnable);
}
public IWorkflowExecutionRunnable getWorkflowExecutionRunnableById(Integer workflowInstanceId) {
return workflowExecutionRunnableMap.get(workflowInstanceId);
}
public Collection<IWorkflowExecutionRunnable> getActiveWorkflowExecutionRunnable() {
return workflowExecutionRunnableMap.values();
}
public void removeWorkflowExecutionRunnable(Integer workflowInstanceId) {
workflowExecutionRunnableMap.remove(workflowInstanceId);
}
}

47
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionContext.java
Unescape View File

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.server.master.events.IEventRepository;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowExecutionContext implements IWorkflowExecutionContext {
private ProcessDefinition workflowDefinition;
private ProcessInstance workflowInstance;
private List<String> beginNodeNames;
private IWorkflowExecutionDAG workflowExecutionDAG;
private IEventRepository eventRepository;
}

31
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionContextFactory.java
Unescape View File

@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.springframework.stereotype.Component;
@Component
public class WorkflowExecutionContextFactory {
public WorkflowExecutionContext createWorkflowExecutionContext(Command command) {
return null;
}
}

116
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionDAG.java
Unescape View File

@ -1,116 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
/**
* The WorkflowExecutionDAG represent a running workflow instance DAG.
*/
@Slf4j
@SuperBuilder
public class WorkflowExecutionDAG implements IWorkflowExecutionDAG {
private final ITaskExecutionContextFactory taskExecutionContextFactory;
private final ITaskExecutionRunnableFactory taskExecutionRunnableFactory;
private final TaskExecutionRunnableRepository taskExecutionRunnableRepository;
private final IWorkflowDAG workflowDAG;
@Override
public TaskExecutionRunnable triggerTask(String taskName) {
DAGNode dagNode = getDAGNode(taskName);
if (dagNode == null) {
throw new IllegalArgumentException("Cannot find the DAGNode for task: " + taskName);
}
// Create task execution context
taskExecutionContextFactory.createTaskExecutionContext();
TaskExecutionRunnable taskExecutionRunnable = taskExecutionRunnableFactory.createTaskExecutionRunnable(null);
taskExecutionRunnableRepository.storeTaskExecutionRunnable(taskExecutionRunnable);
return taskExecutionRunnable;
}
@Override
public TaskExecutionRunnable getTaskExecutionRunnableById(Integer taskInstanceId) {
return taskExecutionRunnableRepository.getTaskExecutionRunnableById(taskInstanceId);
}
@Override
public TaskExecutionRunnable getTaskExecutionRunnableByName(String taskName) {
return taskExecutionRunnableRepository.getTaskExecutionRunnableByName(taskName);
}
@Override
public List<TaskExecutionRunnable> getActiveTaskExecutionRunnable() {
return new ArrayList<>(taskExecutionRunnableRepository.getActiveTaskExecutionRunnable());
}
@Override
public List<TaskExecutionRunnable> getDirectPreTaskExecutionRunnable(String taskName) {
return getDirectPreNodeNames(taskName)
.stream()
.map(taskExecutionRunnableRepository::getTaskExecutionRunnableByName)
.collect(Collectors.toList());
}
@Override
public boolean isTaskAbleToBeTriggered(String taskNodeName) {
// todo: Check whether the workflow instance is finished or ready to finish.
List<DAGNode> directPreNodes = getDirectPreNodes(taskNodeName);
if (log.isDebugEnabled()) {
log.debug("Begin to check whether the task {} is able to be triggered.", taskNodeName);
log.debug("Task {} directly dependent on the task: {}.", taskNodeName,
directPreNodes.stream().map(DAGNode::getNodeName).collect(Collectors.toList()));
}
for (DAGNode directPreNode : directPreNodes) {
if (directPreNode.isSkip()) {
log.debug("The task {} is skipped.", directPreNode.getNodeName());
continue;
}
TaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnableByName(directPreNode.getNodeName());
if (taskExecutionRunnable == null || taskExecutionRunnable.canAccessTo(taskNodeName)) {
log.debug("The task {} is not finished or not able to access to the task {}.",
directPreNode.getNodeName(), taskNodeName);
}
}
return true;
}
@Override
public List<DAGNode> getDirectPostNodes(DAGNode dagNode) {
return workflowDAG.getDirectPostNodes(dagNode);
}
@Override
public List<DAGNode> getDirectPreNodes(DAGNode dagNode) {
return workflowDAG.getDirectPreNodes(dagNode);
}
@Override
public DAGNode getDAGNode(String nodeName) {
return workflowDAG.getDAGNode(nodeName);
}
}

60
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionDAGFactory.java
Unescape View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import java.util.ArrayList;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowExecutionDAGFactory implements IWorkflowExecutionDAGFactory {
@Autowired
private ITaskExecutionRunnableRepositoryFactory taskExecutionRunnableRepositoryFactory;
@Autowired
private ITaskExecutionRunnableFactory taskExecutionRunnableFactory;
@Autowired
private ITaskExecutionContextFactory taskExecutionContextFactory;
@Override
public IWorkflowExecutionDAG createWorkflowExecutionDAG() {
// todo:
TaskExecutionRunnableRepository taskExecutionRunnableRepository =
taskExecutionRunnableRepositoryFactory.createTaskExecutionRunnableRepository();
loadTheHistoryTaskExecutionRunnable()
.forEach(taskExecutionRunnableRepository::storeTaskExecutionRunnable);
return WorkflowExecutionDAG.builder()
.taskExecutionContextFactory(taskExecutionContextFactory)
.taskExecutionRunnableFactory(taskExecutionRunnableFactory)
.taskExecutionRunnableRepository(taskExecutionRunnableRepository)
.workflowDAG(null)
.build();
}
private List<TaskExecutionRunnable> loadTheHistoryTaskExecutionRunnable() {
return new ArrayList<>();
}
}

64
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnable.java
Unescape View File

@ -1,64 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.IEventRepository;
import org.apache.dolphinscheduler.server.master.events.WorkflowTriggeredEvent;
import org.apache.commons.collections4.CollectionUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Getter
@Builder
@AllArgsConstructor
public class WorkflowExecutionRunnable implements IWorkflowExecutionRunnable {
private final IWorkflowExecutionContext workflowExecutionContext;
private final IDAGEngine dagEngine;
public void start() {
if (CollectionUtils.isEmpty(workflowExecutionContext.getBeginNodeNames())) {
dagEngine.triggerNextTasks(null);
} else {
workflowExecutionContext.getBeginNodeNames().forEach(dagEngine::triggerTask);
}
getEventRepository()
.storeEventToTail(new WorkflowTriggeredEvent(workflowExecutionContext.getWorkflowInstance().getId()));
}
@Override
public void pause() {
dagEngine.pauseAllTask();
}
@Override
public void kill() {
dagEngine.killAllTask();
}
@Override
public IEventRepository getEventRepository() {
return workflowExecutionContext.getEventRepository();
}
}

40
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableFactory.java
Unescape View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowExecutionRunnableFactory {
@Autowired
private IDAGEngineFactory dagEngineFactory;
public WorkflowExecutionRunnable createWorkflowExecuteRunnable(IWorkflowExecutionContext workflowExecutionContext) {
IDAGEngine dagEngine = dagEngineFactory.createDAGEngine(workflowExecutionContext);
return WorkflowExecutionRunnable.builder()
.workflowExecutionContext(workflowExecutionContext)
.dagEngine(dagEngine)
.build();
}
}

35
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/WorkflowIdentify.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowIdentify {
private long workflowCode;
private int workflowVersion;
}

111
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/EventEngine.java
Unescape View File

@ -1,111 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionContext;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import org.apache.commons.lang3.time.StopWatch;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class EventEngine extends BaseDaemonThread {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Autowired
private EventFirer eventFirer;
private final Set<Integer> firingWorkflowInstanceIds = ConcurrentHashMap.newKeySet();
public EventEngine() {
super("EventEngine");
}
@Override
public synchronized void start() {
super.start();
log.info(getClass().getName() + " started");
}
@Override
public void run() {
for (;;) {
try {
StopWatch stopWatch = StopWatch.createStarted();
fireAllActiveEvents();
stopWatch.stop();
log.info("Fire all active events cost: {} ms", stopWatch.getTime());
this.wait(5_000);
} catch (Throwable throwable) {
log.error("Fire active event error", throwable);
ThreadUtils.sleep(3_000);
}
}
}
public void fireAllActiveEvents() {
Collection<IWorkflowExecutionRunnable> workflowExecutionRunnableCollection =
workflowExecuteRunnableRepository.getActiveWorkflowExecutionRunnable();
for (IWorkflowExecutionRunnable workflowExecutionRunnable : workflowExecutionRunnableCollection) {
IWorkflowExecutionContext workflowExecutionContext =
workflowExecutionRunnable.getWorkflowExecutionContext();
final Integer workflowInstanceId = workflowExecutionContext.getWorkflowInstanceId();
final String workflowInstanceName = workflowExecutionContext.getWorkflowInstanceName();
try {
LogUtils.setWorkflowInstanceIdMDC(workflowInstanceId);
if (firingWorkflowInstanceIds.contains(workflowInstanceId)) {
log.debug("WorkflowExecutionRunnable: {} is already in firing", workflowInstanceName);
return;
}
IEventRepository workflowEventRepository = workflowExecutionRunnable.getEventRepository();
firingWorkflowInstanceIds.add(workflowInstanceId);
eventFirer.fireActiveEvents(workflowEventRepository)
.whenComplete((fireCount, ex) -> {
firingWorkflowInstanceIds.remove(workflowInstanceId);
if (ex != null) {
log.error("Fire event for WorkflowExecutionRunnable: {} error", workflowInstanceName,
ex);
} else {
if (fireCount > 0) {
log.info("Fire {} events for WorkflowExecutionRunnable: {} success", fireCount,
workflowInstanceName);
}
}
});
} finally {
LogUtils.removeWorkflowInstanceIdMDC();
}
}
}
}

95
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/EventFirer.java
Unescape View File

@ -1,95 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.utils.ExceptionUtils;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class EventFirer implements IEventFirer {
private final IEventOperatorManager<IEvent> eventOperatorManager;
private final ThreadPoolExecutor eventFireThreadPool;
public EventFirer(IEventOperatorManager<IEvent> eventOperatorManager, MasterConfig masterConfig) {
this.eventOperatorManager = eventOperatorManager;
this.eventFireThreadPool =
ThreadUtils.newDaemonFixedThreadExecutor("EventFireThreadPool", masterConfig.getExecThreads());
}
@Override
public CompletableFuture<Integer> fireActiveEvents(IEventRepository eventRepository) {
if (eventRepository.getEventSize() == 0) {
return CompletableFuture.completedFuture(0);
}
return CompletableFuture.supplyAsync(() -> {
int fireCount = 0;
for (;;) {
IEvent event = eventRepository.poolEvent();
if (event == null) {
break;
}
if (event instanceof IAsyncEvent) {
fireAsyncEvent(event);
fireCount++;
continue;
}
try {
fireSyncEvent(event);
fireCount++;
} catch (Exception ex) {
if (ExceptionUtils.isDatabaseConnectedFailedException(ex)) {
// If the event is failed due to cannot connect to DB, we should retry it
eventRepository.storeEventToHead(event);
}
throw ex;
}
}
return fireCount;
}, eventFireThreadPool);
}
private void fireAsyncEvent(IEvent event) {
CompletableFuture.runAsync(() -> {
log.info("Begin fire IAsyncEvent: {}", event);
eventOperatorManager.getEventOperator(event).handleEvent(event);
log.info("Success fire IAsyncEvent: {}", event);
}, eventFireThreadPool).exceptionally(ex -> {
log.error("Failed to fire IAsyncEvent: {}", event, ex);
return null;
});
}
private void fireSyncEvent(IEvent event) {
log.info("Begin fire SyncEvent: {}", event);
eventOperatorManager.getEventOperator(event).handleEvent(event);
log.info("Success fire SyncEvent: {}", event);
}
}

36
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/EventOperatorManager.java
Unescape View File

@ -1,36 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/**
* The event operator manager interface used to get {@link ITaskEventOperator}.
*/
@Slf4j
@Component
public class EventOperatorManager implements IEventOperatorManager<IEvent> {
@Override
public IEventOperator<IEvent> getEventOperator(IEvent event) {
return null;
}
}

24
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IAsyncEvent.java
Unescape View File

@ -1,24 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
/**
* Mark the event as AsyncEvent, if the event is marked as AsyncEvent, the event will be handled asynchronously and we don't .
*/
public interface IAsyncEvent {
}

22
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEvent.java
Unescape View File

@ -1,22 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public interface IEvent {
}

32
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventOperator.java
Unescape View File

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
/**
* The event operator interface used to handle event.
*/
public interface IEventOperator<E> {
/**
* Handle the given event
*
* @param event event
*/
void handleEvent(E event);
}

33
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventOperatorManager.java
Unescape View File

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
/**
* The event operator manager interface used to get event operator.
*/
public interface IEventOperatorManager<E> {
/**
* Get the {@link IEventOperator} for the given event.
*
* @param event event
* @return event operator for the given event
*/
IEventOperator<E> getEventOperator(E event);
}

37
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventRepository.java
Unescape View File

@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import java.util.List;
/**
* The event repository interface used to store event.
*/
public interface IEventRepository {
void storeEventToTail(IEvent event);
void storeEventToHead(IEvent event);
IEvent poolEvent();
int getEventSize();
List<IEvent> getAllEvent();
}

21
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/ISyncEvent.java
Unescape View File

@ -1,21 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public interface ISyncEvent {
}

26
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/ITaskEvent.java
Unescape View File

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public interface ITaskEvent extends IEvent {
Integer getWorkflowInstanceId();
Integer getTaskInstanceId();
}

22
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/ITaskEventOperator.java
Unescape View File

@ -1,22 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public interface ITaskEventOperator<E extends ITaskEvent> extends IEventOperator<E> {
}

29
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IWorkflowEvent.java
Unescape View File

@ -1,29 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public interface IWorkflowEvent extends IEvent {
/**
* The id of WorkflowInstance which the event is related to
*
* @return workflowInstanceId, shouldn't be null
*/
Integer getWorkflowInstanceId();
}

23
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IWorkflowEventOperator.java
Unescape View File

@ -1,23 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public interface IWorkflowEventOperator<E extends IWorkflowEvent>
extends
IEventOperator<E> {
}

60
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/MemoryEventRepository.java
Unescape View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.LinkedBlockingDeque;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MemoryEventRepository implements IEventRepository {
protected final LinkedBlockingDeque<IEvent> eventQueue;
public MemoryEventRepository() {
this.eventQueue = new LinkedBlockingDeque<>();
}
@Override
public void storeEventToTail(IEvent event) {
log.info("Store event to tail: {}", event);
eventQueue.offerLast(event);
}
@Override
public void storeEventToHead(IEvent event) {
eventQueue.offerFirst(event);
}
@Override
public IEvent poolEvent() {
return eventQueue.poll();
}
@Override
public int getEventSize() {
return eventQueue.size();
}
public List<IEvent> getAllEvent() {
return new ArrayList<>(eventQueue);
}
}

36
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskLogSendToRemoteEvent.java
Unescape View File

@ -1,36 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class TaskLogSendToRemoteEvent implements ITaskEvent {
private Integer workflowInstanceId;
private Integer taskInstanceId;
private String taskType;
private String logPath;
}

38
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskLogSendToRemoteEventOperator.java
Unescape View File

@ -1,38 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils;
import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class TaskLogSendToRemoteEventOperator implements ITaskEventOperator<TaskLogSendToRemoteEvent> {
@Override
public void handleEvent(TaskLogSendToRemoteEvent event) {
if (RemoteLogUtils.isRemoteLoggingEnable() && TaskUtils.isMasterTask(event.getTaskType())) {
RemoteLogUtils.sendRemoteLog(event.getLogPath());
log.info("Master sends task log {} to remote storage asynchronously.", event.getLogPath());
}
}
}

46
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskOperationEvent.java
Unescape View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.server.master.dag.TaskExecutionRunnable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class TaskOperationEvent implements ITaskEvent, ISyncEvent {
private TaskExecutionRunnable taskExecutionRunnable;
private TaskOperationType taskOperationType;
@Override
public Integer getWorkflowInstanceId() {
return taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getProcessInstanceId();
}
@Override
public Integer getTaskInstanceId() {
return taskExecutionRunnable.getTaskExecutionContext().getTaskInstance().getId();
}
}

47
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskOperationEventOperator.java
Unescape View File

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.server.master.dag.TaskExecutionRunnable;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class TaskOperationEventOperator implements ITaskEventOperator<TaskOperationEvent> {
@Override
public void handleEvent(TaskOperationEvent event) {
TaskExecutionRunnable taskExecutionRunnable = event.getTaskExecutionRunnable();
switch (event.getTaskOperationType()) {
case RUN:
taskExecutionRunnable.dispatch();
break;
case KILL:
taskExecutionRunnable.kill();
break;
case PAUSE:
taskExecutionRunnable.pause();
break;
default:
log.error("Unknown TaskOperationType for event: {}", event);
}
}
}

29
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskOperationType.java
Unescape View File

@ -1,29 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
public enum TaskOperationType {
FAILOVER,
RUN,
RETRY,
KILL,
PAUSE,
;
}

33
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskSuccessEvent.java
Unescape View File

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TaskSuccessEvent implements ITaskEvent {
private Integer workflowInstanceId;
private Integer taskInstanceId;
}

47
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/TaskSuccessEventOperator.java
Unescape View File

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class TaskSuccessEventOperator implements ITaskEventOperator<TaskSuccessEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Override
public void handleEvent(TaskSuccessEvent event) {
Integer workflowInstanceId = event.getWorkflowInstanceId();
Integer taskInstanceId = event.getTaskInstanceId();
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId);
if (workflowExecutionRunnable == null) {
log.error("Cannot find the WorkflowExecutionRunnable, the event: {} will be dropped", event);
return;
}
}
}

35
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFailedEvent.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowFailedEvent implements IWorkflowEvent, ISyncEvent {
private Integer workflowInstanceId;
private String failedReason;
}

57
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFailedEventOperator.java
Unescape View File

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowFailedEventOperator
implements
IWorkflowEventOperator<WorkflowFailedEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Autowired
private ProcessInstanceDao processInstanceDao;
@Override
public void handleEvent(WorkflowFailedEvent event) {
Integer workflowInstanceId = event.getWorkflowInstanceId();
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId);
ProcessInstance workflowInstance =
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstance();
workflowInstance.setState(WorkflowExecutionStatus.FAILURE);
processInstanceDao.updateById(workflowInstance);
log.info("Handle WorkflowExecutionRunnableFailedEvent success, set workflowInstance status to {}",
workflowInstance.getState());
workflowExecutionRunnable.storeEventToTail(new WorkflowFinalizeEvent(workflowInstanceId));
}
}

32
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFinalizeEvent.java
Unescape View File

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class WorkflowFinalizeEvent implements IWorkflowEvent, ISyncEvent {
private Integer workflowInstanceId;
}

42
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFinalizeEventOperator.java
Unescape View File

@ -1,42 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowFinalizeEventOperator
implements
IWorkflowEventOperator<WorkflowFinalizeEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Override
public void handleEvent(WorkflowFinalizeEvent event) {
Integer workflowInstanceId = event.getWorkflowInstanceId();
workflowExecuteRunnableRepository.removeWorkflowExecutionRunnable(workflowInstanceId);
}
}

37
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowFinishEvent.java
Unescape View File

@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowFinishEvent implements IWorkflowEvent, ISyncEvent {
private Integer workflowInstanceId;
private WorkflowExecutionStatus workflowExecutionStatus;
}

51
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationEvent.java
Unescape View File

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
@AllArgsConstructor
public class WorkflowOperationEvent implements IWorkflowEvent, ISyncEvent {
private Integer workflowInstanceId;
private WorkflowOperationType workflowOperationType;
public static WorkflowOperationEvent of(Integer workflowInstanceId, WorkflowOperationType workflowOperationType) {
return WorkflowOperationEvent.builder()
.workflowInstanceId(workflowInstanceId)
.workflowOperationType(workflowOperationType)
.build();
}
public static WorkflowOperationEvent triggerEvent(Integer workflowInstanceId) {
return of(workflowInstanceId, WorkflowOperationType.TRIGGER);
}
public static WorkflowOperationEvent pauseEvent(Integer workflowInstanceId) {
return of(workflowInstanceId, WorkflowOperationType.PAUSE);
}
public static WorkflowOperationEvent killEvent(Integer workflowInstanceId) {
return of(workflowInstanceId, WorkflowOperationType.KILL);
}
}

87
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationEventOperator.java
Unescape View File

@ -1,87 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionContext;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import org.apache.dolphinscheduler.server.master.utils.ExceptionUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowOperationEventOperator implements IWorkflowEventOperator<WorkflowOperationEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Override
public void handleEvent(WorkflowOperationEvent event) {
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(event.getWorkflowInstanceId());
if (workflowExecutionRunnable == null) {
log.warn(
"Handle workflowExecutionRunnableKillOperationEvent: {} failed: WorkflowExecutionRunnable not found",
event);
return;
}
switch (event.getWorkflowOperationType()) {
case TRIGGER:
triggerWorkflow(workflowExecutionRunnable);
break;
case PAUSE:
pauseWorkflow(workflowExecutionRunnable);
break;
case KILL:
killWorkflow(workflowExecutionRunnable);
break;
default:
log.error("Unknown operationType for event: {}", event);
}
}
private void triggerWorkflow(IWorkflowExecutionRunnable workflowExecutionRunnable) {
try {
workflowExecutionRunnable.start();
} catch (Throwable exception) {
if (ExceptionUtils.isDatabaseConnectedFailedException(exception)) {
throw exception;
}
IWorkflowExecutionContext workflowExecutionContext =
workflowExecutionRunnable.getWorkflowExecutionContext();
log.error("Trigger workflow: {} failed", workflowExecutionContext.getWorkflowInstanceName(), exception);
WorkflowFailedEvent workflowExecutionRunnableFailedEvent = WorkflowFailedEvent.builder()
.workflowInstanceId(workflowExecutionContext.getWorkflowInstanceId())
.failedReason(exception.getMessage())
.build();
workflowExecutionRunnable.storeEventToTail(workflowExecutionRunnableFailedEvent);
}
}
private void pauseWorkflow(IWorkflowExecutionRunnable workflowExecutionRunnable) {
workflowExecutionRunnable.pause();
}
private void killWorkflow(IWorkflowExecutionRunnable workflowExecutionRunnable) {
workflowExecutionRunnable.kill();
}
}

33
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTimeoutEvent.java
Unescape View File

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowTimeoutEvent implements IWorkflowEvent, IAsyncEvent {
private Integer workflowInstanceId;
}

61
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTimeoutEventOperator.java
Unescape View File

@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
import org.apache.dolphinscheduler.service.process.ProcessService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowTimeoutEventOperator
implements
IEventOperator<WorkflowTimeoutEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Autowired
private ProcessService processService;
@Autowired
private ProcessAlertManager processAlertManager;
@Override
public void handleEvent(WorkflowTimeoutEvent event) {
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(event.getWorkflowInstanceId());
if (workflowExecutionRunnable == null) {
log.warn("Cannot find the workflow instance by id: {}", event.getWorkflowInstanceId());
return;
}
// we only support timeout warning for now
ProcessInstance workflowInstance =
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstance();
ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(workflowInstance.getId());
processAlertManager.sendProcessTimeoutAlert(workflowInstance, projectUser);
}
}

38
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggerNextTaskEvent.java
Unescape View File

@ -1,38 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowTriggerNextTaskEvent implements IWorkflowEvent, ISyncEvent {
private Integer workflowInstanceId;
/**
* The task name of the parent task, if it is the root task, the value is null
*/
private String parentTaskNodeName;
}

44
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggerNextTaskEventOperator.java
Unescape View File

@ -1,44 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowTriggerNextTaskEventOperator
implements
IWorkflowEventOperator<WorkflowTriggerNextTaskEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Override
public void handleEvent(WorkflowTriggerNextTaskEvent event) {
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(event.getWorkflowInstanceId());
workflowExecutionRunnable.getDagEngine().triggerNextTasks(event.getParentTaskNodeName());
}
}

33
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggeredEvent.java
Unescape View File

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class WorkflowTriggeredEvent implements IWorkflowEvent, IAsyncEvent {
private Integer workflowInstanceId;
}

63
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowTriggeredEventOperator.java
Unescape View File

@ -1,63 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.server.master.dag.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.dag.WorkflowExecuteRunnableRepository;
import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.process.ProcessService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class WorkflowTriggeredEventOperator
implements
IWorkflowEventOperator<WorkflowTriggeredEvent> {
@Autowired
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
@Autowired
private ProcessService processService;
@Autowired
private ListenerEventAlertManager listenerEventAlertManager;
@Override
public void handleEvent(WorkflowTriggeredEvent event) {
int workflowInstanceId = event.getWorkflowInstanceId();
IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId);
Long workflowDefinitionCode =
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstance()
.getProcessDefinitionCode();
ProcessInstanceMetrics.incProcessInstanceByStateAndProcessDefinitionCode("submit", "" + workflowDefinitionCode);
ProcessInstance workflowInstance =
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstance();
ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(workflowInstanceId);
listenerEventAlertManager.publishProcessStartListenerEvent(workflowInstance, projectUser);
}
}

121
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/DAGNodeTest.java
Unescape View File

@ -1,121 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.ArrayList;
import org.junit.jupiter.api.Test;
import com.google.common.collect.Lists;
class DAGNodeTest {
@Test
void buildDAGNode_EmptyNodeName() {
IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class,
() -> DAGNode.builder()
.inDegrees(new ArrayList<>())
.outDegrees(new ArrayList<>())
.build());
assertEquals("nodeName cannot be empty", illegalArgumentException.getMessage());
}
@Test
void buildDAGNode_BadInDegree() {
IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class,
() -> DAGNode.builder()
.nodeName("A")
.inDegrees(Lists.newArrayList(DAGEdge.builder()
.fromNodeName(null)
.toNodeName("B")
.build()))
.outDegrees(new ArrayList<>())
.build());
assertEquals(
"The toNodeName of inDegree should be the nodeName of the node: A, inDegree: DAGEdge(fromNodeName=null, toNodeName=B)",
illegalArgumentException.getMessage());
}
@Test
void buildDAGNode_NiceInDegree() {
assertDoesNotThrow(() -> DAGNode.builder()
.nodeName("A")
.inDegrees(Lists.newArrayList(DAGEdge.builder()
.fromNodeName(null)
.toNodeName("A")
.build()))
.outDegrees(new ArrayList<>())
.build());
}
@Test
void buildDAGNode_BadOutDegree() {
IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class,
() -> DAGNode.builder()
.nodeName("A")
.inDegrees(new ArrayList<>())
.outDegrees(Lists.newArrayList(DAGEdge.builder()
.fromNodeName("B")
.toNodeName(null)
.build()))
.build());
assertEquals(
"The fromNodeName of outDegree should be the nodeName of the node: A, outDegree: DAGEdge(fromNodeName=B, toNodeName=null)",
illegalArgumentException.getMessage());
}
@Test
void buildDAGNode_NiceOutDegree() {
assertDoesNotThrow(() -> DAGNode.builder()
.nodeName("A")
.inDegrees(new ArrayList<>())
.outDegrees(Lists.newArrayList(DAGEdge.builder()
.fromNodeName("A")
.toNodeName(null)
.build()))
.build());
}
@Test
void buildDAGNode_NotSkip() {
DAGNode dagNode = DAGNode.builder()
.nodeName("A")
.inDegrees(new ArrayList<>())
.outDegrees(new ArrayList<>())
.build();
assertFalse(dagNode.isSkip());
}
@Test
void buildDAGNode_Skip() {
DAGNode dagNode = DAGNode.builder()
.nodeName("A")
.skip(true)
.inDegrees(new ArrayList<>())
.outDegrees(new ArrayList<>())
.build();
assertTrue(dagNode.isSkip());
}
}

88
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/MockWorkflowExecutionContextFactory.java
Unescape View File

@ -1,88 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.server.master.events.MemoryEventRepository;
import java.util.Collections;
class MockWorkflowExecutionContextFactory {
public static WorkflowExecutionContext createWorkflowExecutionContext() {
long workflowDefinitionCode = CodeGenerateUtils.getInstance().genCode();
int workflowDefinitionVersion = 1;
int workflowInstanceId = 1;
String workflowInstanceName = "MockWorkflow" + CodeGenerateUtils.getInstance().genCode();
ProcessDefinition processDefinition = ProcessDefinition.builder()
.id(1)
.name("TestWorkflow")
.code(workflowDefinitionCode)
.version(workflowDefinitionVersion)
.build();
ProcessInstance processInstance = ProcessInstance.builder()
.id(workflowInstanceId)
.name(workflowInstanceName)
.build();
WorkflowExecutionDAG workflowExecutionDAG = createWorkflowExecutionDAG();
return WorkflowExecutionContext.builder()
.workflowExecutionDAG(workflowExecutionDAG)
.eventRepository(new MemoryEventRepository())
.beginNodeNames(Collections.emptyList())
.workflowDefinition(processDefinition)
.workflowInstance(processInstance)
.build();
}
private static WorkflowExecutionDAG createWorkflowExecutionDAG() {
TaskDefinitionLog taskA = taskNode("A");
WorkflowDAG workflowDAG = WorkflowDAGBuilder.newBuilder()
.addTaskNode(taskA)
.addTaskEdge(edge(null, taskA))
.addTaskEdge(edge(taskA, null))
.build();
return WorkflowExecutionDAG.builder()
.workflowDAG(workflowDAG)
.taskExecutionRunnableRepository(new TaskExecutionRunnableRepository())
.build();
}
private static TaskDefinitionLog taskNode(String nodeName) {
TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog();
taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode());
taskDefinitionLog.setName(nodeName);
return taskDefinitionLog;
}
private static ProcessTaskRelationLog edge(TaskDefinitionLog from, TaskDefinitionLog to) {
ProcessTaskRelationLog processTaskRelationLog = new ProcessTaskRelationLog();
processTaskRelationLog.setPreTaskCode(from == null ? 0 : from.getCode());
processTaskRelationLog.setPostTaskCode(to == null ? 0 : to.getCode());
return processTaskRelationLog;
}
}

31
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/MockWorkflowExecutionRunnableFactory.java
Unescape View File

@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
class MockWorkflowExecutionRunnableFactory {
public static WorkflowExecutionRunnable createWorkflowExecuteRunnable() {
WorkflowExecutionContext workflowExecutionContext =
MockWorkflowExecutionContextFactory.createWorkflowExecutionContext();
IDAGEngine dagEngine = new DAGEngine(workflowExecutionContext);
return WorkflowExecutionRunnable.builder()
.workflowExecutionContext(workflowExecutionContext)
.dagEngine(dagEngine)
.build();
}
}

171
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGAssertion.java
Unescape View File

@ -1,171 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.commons.collections4.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
public class WorkflowDAGAssertion {
private final IWorkflowDAG workflowDAG;
private final List<NodeAssertion> nodeAssertions;
private WorkflowDAGAssertion(IWorkflowDAG workflowDAG) {
this.workflowDAG = workflowDAG;
this.nodeAssertions = new ArrayList<>();
}
public static WorkflowDAGAssertion workflowDag(IWorkflowDAG workflowDAG) {
return new WorkflowDAGAssertion(workflowDAG);
}
public WorkflowDAGAssertion nodeAssertion(NodeAssertion nodeAssertion) {
nodeAssertions.add(nodeAssertion);
return this;
}
public void doAssertion() {
nodeAssertions.forEach(nodeAssertion -> nodeAssertion.doAssertion(workflowDAG));
}
public static class NodeAssertion {
/**
* node name of the assertion
*/
private final String nodeName;
/**
* whether the node exist
*/
private boolean exist;
/**
* whether the node is skipped
*/
private boolean skip;
/**
* whether the node has out degree
*/
private List<String> postNodes;
/**
* whether the node has in degree
*/
private List<String> preNodes;
private NodeAssertion(String nodeName) {
this.nodeName = nodeName;
this.postNodes = new ArrayList<>();
this.preNodes = new ArrayList<>();
}
public static NodeAssertion node(String nodeName) {
return new NodeAssertion(nodeName);
}
public NodeAssertion exist() {
this.exist = true;
return this;
}
public NodeAssertion skip() {
this.skip = true;
return this;
}
public NodeAssertion noEdge() {
this.postNodes = new ArrayList<>();
this.preNodes = new ArrayList<>();
return this;
}
public NodeAssertion noInDegree() {
this.preNodes = new ArrayList<>();
return this;
}
public NodeAssertion inDegrees(List<String> preNodes) {
this.preNodes.addAll(preNodes);
return this;
}
public NodeAssertion inDegrees(String preNode) {
this.preNodes.add(preNode);
return this;
}
public NodeAssertion noOutDegree() {
this.postNodes = new ArrayList<>();
return this;
}
public NodeAssertion outDegrees(List<String> postNodes) {
this.postNodes.addAll(postNodes);
return this;
}
public NodeAssertion outDegrees(String postNode) {
this.postNodes.add(postNode);
return this;
}
private void doAssertion(IWorkflowDAG workflowDAG) {
if (exist) {
// node exist
assertNotNull(workflowDAG.getDAGNode(nodeName), "node " + nodeName + " does not exist");
} else {
assertNull(workflowDAG.getDAGNode(nodeName), "node " + nodeName + " exist");
}
if (skip) {
assertTrue(workflowDAG.getDAGNode(nodeName).isSkip(), "node " + nodeName + " is not skipped");
} else {
assertFalse(workflowDAG.getDAGNode(nodeName).isSkip(), "node " + nodeName + " is skipped");
}
if (CollectionUtils.isEmpty(postNodes)) {
assertTrue(workflowDAG.getDirectPostNodeNames(nodeName).isEmpty(),
"node " + nodeName + " has outDegree " + workflowDAG.getDirectPostNodes(nodeName));
} else {
postNodes
.forEach(postNode -> assertTrue(workflowDAG.getDirectPostNodeNames(nodeName).contains(postNode),
"node " + nodeName + " does has outDegree " + postNode));
}
if (CollectionUtils.isEmpty(preNodes)) {
assertTrue(workflowDAG.getDirectPreNodeNames(nodeName).isEmpty(),
"node " + nodeName + " has inDegree " + workflowDAG.getDirectPreNodeNames(nodeName));
} else {
preNodes.forEach(preNode -> assertTrue(workflowDAG.getDirectPreNodeNames(nodeName).contains(preNode),
"node " + nodeName + " does has inDegree " + preNode));
}
}
}
}

282
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowDAGBuilderTest.java
Unescape View File

@ -1,282 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import static com.google.common.collect.Lists.newArrayList;
import static org.apache.dolphinscheduler.server.master.dag.WorkflowDAGAssertion.NodeAssertion.node;
import static org.apache.dolphinscheduler.server.master.dag.WorkflowDAGAssertion.workflowDag;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.junit.jupiter.api.Test;
class WorkflowDAGBuilderTest {
/**
* Test DAG with single node:
* <pre>
* {@code
* Node(A)
* }
* </pre>
*/
@Test
void build_SingleTaskNode() {
String nodeName = "A";
TaskDefinitionLog taskDefinitionLog = taskNode(nodeName);
WorkflowDAG workflowDAG = WorkflowDAGBuilder.newBuilder()
.addTaskNode(taskDefinitionLog)
.build();
workflowDag(workflowDAG)
.nodeAssertion(node(nodeName).exist().noEdge())
.doAssertion();
}
/**
* Test DAG with multiple nodes:
* <pre>
* {@code
* Node(A)
* Node(B)
* Node(C)
* }
* <pre/>
*/
@Test
void build_MULTIPLE_NODE() {
String nodeNameA = "A";
TaskDefinitionLog taskNodeA = taskNode(nodeNameA);
String nodeNameB = "B";
TaskDefinitionLog taskNodeB = taskNode(nodeNameB);
String nodeNameC = "C";
TaskDefinitionLog taskNodeC = taskNode(nodeNameC);
WorkflowDAG workflowDAG = WorkflowDAGBuilder.newBuilder()
.addTaskNode(taskNodeA)
.addTaskNode(taskNodeB)
.addTaskNode(taskNodeC)
.build();
workflowDag(workflowDAG)
.nodeAssertion(node(nodeNameA).exist().noEdge())
.nodeAssertion(node(nodeNameB).exist().noEdge())
.nodeAssertion(node(nodeNameC).exist().noEdge())
.doAssertion();
}
/**
* Test DAG with multiple nodes:
* <pre>
* {@code
* Node(A) -> Node(B1) -> Node(C1) -> Node(D)
* -> Node(B2) -> Node(C2) ->
* }
* <pre/>
*/
@Test
void build_DAG() {
String nodeNameA = "A";
TaskDefinitionLog taskNodeA = taskNode(nodeNameA);
String nodeNameB1 = "B1";
TaskDefinitionLog taskNodeB1 = taskNode(nodeNameB1);
String nodeNameB2 = "B2";
TaskDefinitionLog taskNodeB2 = taskNode(nodeNameB2);
String nodeNameC1 = "C1";
TaskDefinitionLog taskNodeC1 = taskNode(nodeNameC1);
String nodeNameC2 = "C2";
TaskDefinitionLog taskNodeC2 = taskNode(nodeNameC2);
String nodeNameD = "D";
TaskDefinitionLog taskNodeD = taskNode(nodeNameD);
WorkflowDAG workflowDAG = WorkflowDAGBuilder.newBuilder()
.addTaskNode(taskNodeA)
.addTaskNodes(newArrayList(taskNodeB1, taskNodeB2))
.addTaskNodes(newArrayList(taskNodeC1, taskNodeC2))
.addTaskNode(taskNodeD)
.addTaskEdge(edge(null, taskNodeA))
.addTaskEdge(edge(taskNodeA, taskNodeB1))
.addTaskEdge(edge(taskNodeA, taskNodeB2))
.addTaskEdge(edge(taskNodeB1, taskNodeC1))
.addTaskEdge(edge(taskNodeB2, taskNodeC2))
.addTaskEdge(edge(taskNodeC1, taskNodeD))
.addTaskEdge(edge(taskNodeC2, taskNodeD))
.addTaskEdge(edge(taskNodeD, null))
.build();
workflowDag(workflowDAG)
.nodeAssertion(node(nodeNameA)
.exist()
.noInDegree()
.outDegrees(newArrayList(nodeNameB1, nodeNameB2)))
.nodeAssertion(node(nodeNameB1)
.exist()
.inDegrees(nodeNameA)
.outDegrees(nodeNameC1))
.nodeAssertion(node(nodeNameB2)
.exist()
.inDegrees(nodeNameA)
.outDegrees(nodeNameC2))
.nodeAssertion(node(nodeNameC1)
.exist()
.inDegrees(nodeNameB1)
.outDegrees(nodeNameD))
.nodeAssertion(node(nodeNameC2)
.exist()
.inDegrees(nodeNameB2)
.outDegrees(nodeNameD))
.nodeAssertion(node(nodeNameD)
.exist()
.inDegrees(newArrayList(nodeNameC1, nodeNameC2))
.noOutDegree())
.doAssertion();
}
/**
* Test DAG with multiple sub dags:
* <pre>
* {@code
* Node(A1) -> Node(B1) -> Node(C1) -> Node(D1)
* -> Node(B2) -> Node(C2) ->
*
* Node(A2) -> Node(B3) -> Node(C3) -> Node(D2)
* -> Node(B4) -> Node(C4) ->
* }
* <pre/>
*/
@Test
void build_MULTIPLE_SUB_DAG() {
String nodeNameA1 = "A1";
TaskDefinitionLog taskNodeA1 = taskNode(nodeNameA1);
String nodeNameA2 = "A2";
TaskDefinitionLog taskNodeA2 = taskNode(nodeNameA2);
String nodeNameB1 = "B1";
TaskDefinitionLog taskNodeB1 = taskNode(nodeNameB1);
String nodeNameB2 = "B2";
TaskDefinitionLog taskNodeB2 = taskNode(nodeNameB2);
String nodeNameB3 = "B3";
TaskDefinitionLog taskNodeB3 = taskNode(nodeNameB3);
String nodeNameB4 = "B4";
TaskDefinitionLog taskNodeB4 = taskNode(nodeNameB4);
String nodeNameC1 = "C1";
TaskDefinitionLog taskNodeC1 = taskNode(nodeNameC1);
String nodeNameC2 = "C2";
TaskDefinitionLog taskNodeC2 = taskNode(nodeNameC2);
String nodeNameC3 = "C3";
TaskDefinitionLog taskNodeC3 = taskNode(nodeNameC3);
String nodeNameC4 = "C4";
TaskDefinitionLog taskNodeC4 = taskNode(nodeNameC4);
String nodeNameD1 = "D1";
TaskDefinitionLog taskNodeD1 = taskNode(nodeNameD1);
String nodeNameD2 = "D2";
TaskDefinitionLog taskNodeD2 = taskNode(nodeNameD2);
WorkflowDAG workflowDAG = WorkflowDAGBuilder.newBuilder()
.addTaskNodes(newArrayList(taskNodeA1, taskNodeA2))
.addTaskNodes(newArrayList(taskNodeB1, taskNodeB2, taskNodeB3, taskNodeB4))
.addTaskNodes(newArrayList(taskNodeC1, taskNodeC2, taskNodeC3, taskNodeC4))
.addTaskNodes(newArrayList(taskNodeD1, taskNodeD2))
.addTaskEdge(edge(null, taskNodeA1))
.addTaskEdge(edge(taskNodeA1, taskNodeB1))
.addTaskEdge(edge(taskNodeA1, taskNodeB2))
.addTaskEdge(edge(taskNodeB1, taskNodeC1))
.addTaskEdge(edge(taskNodeB2, taskNodeC2))
.addTaskEdge(edge(taskNodeC1, taskNodeD1))
.addTaskEdge(edge(taskNodeC2, taskNodeD1))
.addTaskEdge(edge(taskNodeD1, null))
.addTaskEdge(edge(null, taskNodeA2))
.addTaskEdge(edge(taskNodeA2, taskNodeB3))
.addTaskEdge(edge(taskNodeA2, taskNodeB4))
.addTaskEdge(edge(taskNodeB3, taskNodeC3))
.addTaskEdge(edge(taskNodeB4, taskNodeC4))
.addTaskEdge(edge(taskNodeC3, taskNodeD2))
.addTaskEdge(edge(taskNodeC4, taskNodeD2))
.addTaskEdge(edge(taskNodeD2, null))
.build();
workflowDag(workflowDAG)
.nodeAssertion(node(nodeNameA1)
.exist()
.noInDegree()
.outDegrees(newArrayList(nodeNameB1, nodeNameB2)))
.nodeAssertion(node(nodeNameB1)
.exist()
.inDegrees(nodeNameA1)
.outDegrees(nodeNameC1))
.nodeAssertion(node(nodeNameB2)
.exist()
.inDegrees(nodeNameA1)
.outDegrees(nodeNameC2))
.nodeAssertion(node(nodeNameC1)
.exist()
.inDegrees(nodeNameB1)
.outDegrees(nodeNameD1))
.nodeAssertion(node(nodeNameC2)
.exist()
.inDegrees(nodeNameB2)
.outDegrees(nodeNameD1))
.nodeAssertion(node(nodeNameD1)
.exist()
.inDegrees(newArrayList(nodeNameC1, nodeNameC2))
.noOutDegree())
.doAssertion();
workflowDag(workflowDAG)
.nodeAssertion(node(nodeNameA2)
.exist()
.noInDegree()
.outDegrees(newArrayList(nodeNameB3, nodeNameB4)))
.nodeAssertion(node(nodeNameB3)
.exist()
.inDegrees(nodeNameA2)
.outDegrees(nodeNameC3))
.nodeAssertion(node(nodeNameB4)
.exist()
.inDegrees(nodeNameA2)
.outDegrees(nodeNameC4))
.nodeAssertion(node(nodeNameC3)
.exist()
.inDegrees(nodeNameB3)
.outDegrees(nodeNameD2))
.nodeAssertion(node(nodeNameC4)
.exist()
.inDegrees(nodeNameB4)
.outDegrees(nodeNameD2))
.nodeAssertion(node(nodeNameD2)
.exist()
.inDegrees(newArrayList(nodeNameC3, nodeNameC4))
.noOutDegree())
.doAssertion();
}
private TaskDefinitionLog taskNode(String nodeName) {
TaskDefinitionLog taskDefinitionLog = new TaskDefinitionLog();
taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode());
taskDefinitionLog.setName(nodeName);
return taskDefinitionLog;
}
private ProcessTaskRelationLog edge(TaskDefinitionLog from, TaskDefinitionLog to) {
ProcessTaskRelationLog processTaskRelationLog = new ProcessTaskRelationLog();
processTaskRelationLog.setPreTaskCode(from == null ? 0 : from.getCode());
processTaskRelationLog.setPostTaskCode(to == null ? 0 : to.getCode());
return processTaskRelationLog;
}
}

114
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowEngineTest.java
Unescape View File

@ -1,114 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import static org.apache.dolphinscheduler.server.master.dag.WorkflowExecutionRunnableAssertions.workflowExecutionRunnable;
import static org.apache.dolphinscheduler.server.master.dag.WorkflowExecutionRunnableRepositoryAssertions.workflowExecutionRunnableRepository;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.apache.dolphinscheduler.server.master.events.WorkflowOperationEvent;
import org.apache.dolphinscheduler.server.master.exception.WorkflowExecuteRunnableNotFoundException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class WorkflowEngineTest {
private WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
private WorkflowEngine workflowEngine;
@BeforeEach
public void before() {
workflowExecuteRunnableRepository = new WorkflowExecuteRunnableRepository();
workflowEngine = new WorkflowEngine(workflowExecuteRunnableRepository);
}
@Test
void triggerWorkflow() {
IWorkflowExecutionRunnable emptyWorkflowExecuteRunnable =
MockWorkflowExecutionRunnableFactory.createWorkflowExecuteRunnable();
Integer workflowInstanceId = emptyWorkflowExecuteRunnable.getWorkflowExecutionContext().getWorkflowInstanceId();
workflowEngine.triggerWorkflow(emptyWorkflowExecuteRunnable);
workflowExecutionRunnable(emptyWorkflowExecuteRunnable)
.existEvent(WorkflowOperationEvent.triggerEvent(workflowInstanceId));
}
@Test
void pauseWorkflow_WorkflowNotExist() {
WorkflowExecuteRunnableNotFoundException exception =
assertThrows(WorkflowExecuteRunnableNotFoundException.class, () -> workflowEngine.pauseWorkflow(1));
assertEquals("WorkflowExecuteRunnable not found: [id=1]", exception.getMessage());
}
@Test
void pauseWorkflow_WorkflowExist() {
IWorkflowExecutionRunnable emptyWorkflowExecuteRunnable =
MockWorkflowExecutionRunnableFactory.createWorkflowExecuteRunnable();
Integer workflowInstanceId =
emptyWorkflowExecuteRunnable.getWorkflowExecutionContext().getWorkflowInstanceId();
workflowExecuteRunnableRepository.storeWorkflowExecutionRunnable(emptyWorkflowExecuteRunnable);
workflowEngine.pauseWorkflow(workflowInstanceId);
workflowExecutionRunnable(emptyWorkflowExecuteRunnable)
.existEvent(WorkflowOperationEvent.pauseEvent(workflowInstanceId));
}
@Test
void killWorkflow_WorkflowNotExist() {
WorkflowExecuteRunnableNotFoundException exception =
assertThrows(WorkflowExecuteRunnableNotFoundException.class,
() -> workflowEngine.killWorkflow(1));
assertEquals("WorkflowExecuteRunnable not found: [id=1]", exception.getMessage());
}
@Test
void killWorkflow_WorkflowExist() {
IWorkflowExecutionRunnable emptyWorkflowExecuteRunnable =
MockWorkflowExecutionRunnableFactory.createWorkflowExecuteRunnable();
Integer workflowInstanceId =
emptyWorkflowExecuteRunnable.getWorkflowExecutionContext().getWorkflowInstanceId();
workflowExecuteRunnableRepository.storeWorkflowExecutionRunnable(emptyWorkflowExecuteRunnable);
workflowEngine.killWorkflow(workflowInstanceId);
workflowExecutionRunnable(emptyWorkflowExecuteRunnable)
.existEvent(WorkflowOperationEvent.killEvent(workflowInstanceId));
}
@Test
void finalizeWorkflow_WorkflowNotExist() {
workflowEngine.finalizeWorkflow(1);
}
@Test
void finalizeWorkflow_WorkflowExist() {
IWorkflowExecutionRunnable emptyWorkflowExecuteRunnable =
MockWorkflowExecutionRunnableFactory.createWorkflowExecuteRunnable();
Integer workflowInstanceId =
emptyWorkflowExecuteRunnable.getWorkflowExecutionContext().getWorkflowInstanceId();
workflowExecuteRunnableRepository.storeWorkflowExecutionRunnable(emptyWorkflowExecuteRunnable);
workflowExecutionRunnableRepository(workflowExecuteRunnableRepository)
.existWorkflowExecutionRunnable(workflowInstanceId);
workflowEngine.finalizeWorkflow(workflowInstanceId);
workflowExecutionRunnableRepository(workflowExecuteRunnableRepository)
.notExistWorkflowExecutionRunnable(workflowInstanceId);
}
}

48
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableAssertions.java
Unescape View File

@ -1,48 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import org.apache.dolphinscheduler.server.master.events.IEvent;
import org.opentest4j.AssertionFailedError;
class WorkflowExecutionRunnableAssertions {
private final IWorkflowExecutionRunnable workflowExecutionRunnable;
private WorkflowExecutionRunnableAssertions(IWorkflowExecutionRunnable workflowExecutionRunnable) {
this.workflowExecutionRunnable = workflowExecutionRunnable;
}
public static WorkflowExecutionRunnableAssertions workflowExecutionRunnable(IWorkflowExecutionRunnable workflowExecutionRunnable) {
return new WorkflowExecutionRunnableAssertions(workflowExecutionRunnable);
}
public void existEvent(IEvent event) {
if (event == null) {
throw new IllegalArgumentException("Event cannot be null");
}
boolean exist = workflowExecutionRunnable.getEventRepository().getAllEvent()
.stream()
.anyMatch(event1 -> event1.equals(event1));
if (!exist) {
throw new AssertionFailedError("The workflowExecuteRunnable doesn't exist event: " + event);
}
}
}

43
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableRepositoryAssertions.java
Unescape View File

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
class WorkflowExecutionRunnableRepositoryAssertions {
private final WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
private WorkflowExecutionRunnableRepositoryAssertions(WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository) {
this.workflowExecuteRunnableRepository = workflowExecuteRunnableRepository;
}
public static WorkflowExecutionRunnableRepositoryAssertions workflowExecutionRunnableRepository(WorkflowExecuteRunnableRepository workflowExecuteRunnableRepository) {
return new WorkflowExecutionRunnableRepositoryAssertions(workflowExecuteRunnableRepository);
}
public void existWorkflowExecutionRunnable(Integer workflowInstanceId) {
assertNotNull(workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId));
}
public void notExistWorkflowExecutionRunnable(Integer workflowInstanceId) {
assertNull(workflowExecuteRunnableRepository.getWorkflowExecutionRunnableById(workflowInstanceId));
}
}

51
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowExecutionRunnableTest.java
Unescape View File

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
import static org.apache.dolphinscheduler.server.master.dag.WorkflowExecutionRunnableAssertions.workflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.events.WorkflowTriggeredEvent;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class WorkflowExecutionRunnableTest {
private WorkflowExecutionRunnable workflowExecutionRunnable;
@BeforeEach
public void before() {
workflowExecutionRunnable = MockWorkflowExecutionRunnableFactory.createWorkflowExecuteRunnable();
}
@Test
void start() {
workflowExecutionRunnable.start();
workflowExecutionRunnable(workflowExecutionRunnable)
.existEvent(new WorkflowTriggeredEvent(
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstance().getId()));
}
@Test
void pause() {
}
@Test
void kill() {
}
}

46
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationEventTest.java
Unescape View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
class WorkflowOperationEventTest {
@Test
void triggerEvent() {
WorkflowOperationEvent workflowOperationEvent = WorkflowOperationEvent.triggerEvent(1);
assertEquals(1, workflowOperationEvent.getWorkflowInstanceId());
assertEquals(WorkflowOperationType.TRIGGER, workflowOperationEvent.getWorkflowOperationType());
}
@Test
void pauseEvent() {
WorkflowOperationEvent workflowOperationEvent = WorkflowOperationEvent.pauseEvent(1);
assertEquals(1, workflowOperationEvent.getWorkflowInstanceId());
assertEquals(WorkflowOperationType.PAUSE, workflowOperationEvent.getWorkflowOperationType());
}
@Test
void killEvent() {
WorkflowOperationEvent workflowOperationEvent = WorkflowOperationEvent.killEvent(1);
assertEquals(1, workflowOperationEvent.getWorkflowInstanceId());
assertEquals(WorkflowOperationType.KILL, workflowOperationEvent.getWorkflowOperationType());
}
}

66
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/EventEngine.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/EventEngine.java
Unescape View File

@ -15,57 +15,59 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.workflow.engine.event;
package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecuteRunnableRepository;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionContext;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnableRepository;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.time.StopWatch;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.MDC;
@Slf4j
public class EventEngine extends BaseDaemonThread {
public class EventEngine extends BaseDaemonThread implements IEventEngine {
private final IWorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
private final IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository;
private final EventFirer eventFirer;
private final IEventFirer eventFirer;
private final Set<Integer> firingWorkflowInstanceIds;
private volatile boolean stop = false;
public EventEngine(IWorkflowExecuteRunnableRepository workflowExecuteRunnableRepository,
EventFirer eventFirer) {
public EventEngine(IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository,
IEventFirer eventFirer) {
super("EventEngine");
this.workflowExecuteRunnableRepository = workflowExecuteRunnableRepository;
this.eventFirer = eventFirer;
this.firingWorkflowInstanceIds = ConcurrentHashMap.newKeySet();
}
@Override
public synchronized void start() {
this.stop = false;
super.start();
log.info(getClass().getName() + " started");
}
@Override
public void run() {
for (;;) {
while (!stop) {
try {
StopWatch stopWatch = StopWatch.createStarted();
fireAllActiveEvents();
stopWatch.stop();
log.info("Fire all active events cost: {} ms", stopWatch.getTime());
this.wait(5_000);
Collection<IWorkflowExecutionRunnable> workflowExecutionRunnableCollection =
workflowExecuteRunnableRepository.getActiveWorkflowExecutionRunnable();
if (CollectionUtils.isEmpty(workflowExecutionRunnableCollection)) {
log.debug("There is no active WorkflowExecutionRunnable");
this.wait(3_000);
continue;
}
fireAllActiveEvents(workflowExecutionRunnableCollection);
} catch (Throwable throwable) {
log.error("Fire active event error", throwable);
ThreadUtils.sleep(3_000);
@ -73,25 +75,25 @@ public class EventEngine extends BaseDaemonThread {
}
}
public void fireAllActiveEvents() {
Collection<IWorkflowExecutionRunnable> workflowExecutionRunnableCollection =
workflowExecuteRunnableRepository.getActiveWorkflowExecutionRunnable();
for (IWorkflowExecutionRunnable workflowExecutionRunnable : workflowExecutionRunnableCollection) {
public void fireAllActiveEvents(Collection<IWorkflowExecutionRunnable> workflowExecutionRunnableList) {
StopWatch stopWatch = StopWatch.createStarted();
log.info("Fire all active events cost: {} ms", stopWatch.getTime());
stopWatch.stop();
for (IWorkflowExecutionRunnable workflowExecutionRunnable : workflowExecutionRunnableList) {
IWorkflowExecutionContext workflowExecutionContext =
workflowExecutionRunnable.getWorkflowExecutionContext();
final Integer workflowInstanceId = workflowExecutionContext.getWorkflowInstanceId();
final String workflowInstanceName = workflowExecutionContext.getWorkflowInstanceName();
try {
MDC.put(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY, String.valueOf(workflowInstanceId));
if (firingWorkflowInstanceIds.contains(workflowInstanceId)) {
if (workflowExecutionRunnable.isEventFiring()) {
log.debug("WorkflowExecutionRunnable: {} is already in firing", workflowInstanceName);
return;
continue;
}
IEventRepository workflowEventRepository = workflowExecutionRunnable.getEventRepository();
firingWorkflowInstanceIds.add(workflowInstanceId);
eventFirer.fireActiveEvents(workflowEventRepository)
eventFirer.fireActiveEvents(workflowExecutionRunnable)
.whenComplete((fireCount, ex) -> {
firingWorkflowInstanceIds.remove(workflowInstanceId);
workflowExecutionRunnable.setEventFiring(false);
if (ex != null) {
log.error("Fire event for WorkflowExecutionRunnable: {} error", workflowInstanceName,
ex);
@ -108,4 +110,14 @@ public class EventEngine extends BaseDaemonThread {
}
}
@Override
public synchronized void shutdown() {
if (stop) {
log.warn("EventEngine has already stopped");
return;
}
this.stop = true;
eventFirer.shutdown();
}
}

72
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/EventEngineFactory.java
Unescape View File

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.workflow.engine.event.EventOperatorManager;
import org.apache.dolphinscheduler.workflow.engine.event.IEvent;
import org.apache.dolphinscheduler.workflow.engine.event.IEventOperatorManager;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnableRepository;
import org.apache.dolphinscheduler.workflow.engine.workflow.SingletonWorkflowExecutionRunnableRepository;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class EventEngineFactory implements IEventEngineFactory {
private static final IWorkflowExecutionRunnableRepository DEFAULT_WORKFLOW_EXECUTION_RUNNABLE_FACTORY =
SingletonWorkflowExecutionRunnableRepository.getInstance();
private IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository;
private static final IEventOperatorManager<IEvent> DEFAULT_EVENT_OPERATOR_MANAGER =
EventOperatorManager.getInstance();
private IEventOperatorManager<IEvent> eventOperatorManager = DEFAULT_EVENT_OPERATOR_MANAGER;
private static final int DEFAULT_EVENT_FIRE_THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2;
private int eventFireThreadPoolSize = DEFAULT_EVENT_FIRE_THREAD_POOL_SIZE;
private EventEngineFactory() {
}
public static EventEngineFactory newEventEngineFactory() {
return new EventEngineFactory();
}
public EventEngineFactory withWorkflowExecuteRunnableRepository(IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository) {
this.workflowExecuteRunnableRepository = workflowExecuteRunnableRepository;
return this;
}
public EventEngineFactory withEventOperatorManager(IEventOperatorManager<IEvent> eventOperatorManager) {
this.eventOperatorManager = eventOperatorManager;
return this;
}
public int withEventFireThreadPoolSize(int eventFireThreadPoolSize) {
this.eventFireThreadPoolSize = eventFireThreadPoolSize;
return this.eventFireThreadPoolSize;
}
@Override
public IEventEngine createEventEngine() {
EventFirer eventFirer = new EventFirer(eventOperatorManager, eventFireThreadPoolSize);
return new EventEngine(workflowExecuteRunnableRepository, eventFirer);
}
}

16
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/EventFirer.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/EventFirer.java
Unescape View File

@ -15,10 +15,15 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.workflow.engine.event;
package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.workflow.engine.event.IAsyncEvent;
import org.apache.dolphinscheduler.workflow.engine.event.IEvent;
import org.apache.dolphinscheduler.workflow.engine.event.IEventOperatorManager;
import org.apache.dolphinscheduler.workflow.engine.event.IEventRepository;
import org.apache.dolphinscheduler.workflow.engine.utils.ExceptionUtils;
import org.apache.dolphinscheduler.workflow.engine.workflow.IEventfulExecutionRunnable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
@ -39,7 +44,9 @@ public class EventFirer implements IEventFirer {
}
@Override
public CompletableFuture<Integer> fireActiveEvents(IEventRepository eventRepository) {
public CompletableFuture<Integer> fireActiveEvents(IEventfulExecutionRunnable eventfulExecutionRunnable) {
// todo: add MDC key
IEventRepository eventRepository = eventfulExecutionRunnable.getEventRepository();
if (eventRepository.getEventSize() == 0) {
return CompletableFuture.completedFuture(0);
}
@ -71,6 +78,11 @@ public class EventFirer implements IEventFirer {
}, eventFireThreadPool);
}
@Override
public void shutdown() {
eventFireThreadPool.shutdown();
}
private void fireAsyncEvent(IEvent event) {
CompletableFuture.runAsync(() -> {
log.info("Begin fire IAsyncEvent: {}", event);

8
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IDelayEvent.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventAcceptor.java
Unescape View File

@ -15,12 +15,12 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
package org.apache.dolphinscheduler.workflow.engine.engine;
public interface IDelayEvent {
import org.apache.dolphinscheduler.workflow.engine.event.IEvent;
long getEventCreateTime();
public interface IEventAcceptor {
long getDelayTime();
void accept(IEvent event);
}

8
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dag/ITaskExecutionContext.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventDispatcher.java
Unescape View File

@ -15,12 +15,12 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.workflow.engine.event.IEvent;
public interface ITaskExecutionContext {
public interface IEventDispatcher {
TaskInstance getTaskInstance();
void dispatch(IEvent event);
}

18
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/WorkflowOperationType.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventEngine.java
Unescape View File

@ -15,22 +15,18 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
package org.apache.dolphinscheduler.workflow.engine.engine;
public enum WorkflowOperationType {
public interface IEventEngine {
/**
* Trigger the workflow instance.
* Start the event engine.
*/
TRIGGER,
/**
* Pause the workflow instance, it will pause the running task instances.
*/
PAUSE,
void start();
/**
* Kill the workflow instance, it will kill the running task instances.
* Shutdown the event engine. The event engine cannot be restarted after shutdown. This method will block until the event engine is completely shutdown.
*/
KILL,
;
void shutdown();
}

6
dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dag/WorkflowEngineIT.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventEngineFactory.java
Unescape View File

@ -15,8 +15,10 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.dag;
package org.apache.dolphinscheduler.workflow.engine.engine;
public class WorkflowEngineIT {
public interface IEventEngineFactory {
IEventEngine createEventEngine();
}

13
dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/events/IEventFirer.java → dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IEventFirer.java
Unescape View File

@ -15,12 +15,14 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master.events;
package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.workflow.engine.workflow.IEventfulExecutionRunnable;
import java.util.concurrent.CompletableFuture;
/**
* The event firer interface used to fire event.
* The event firer interface used to fire event for {@link IEventfulExecutionRunnable}.
*
*/
public interface IEventFirer {
@ -30,6 +32,11 @@ public interface IEventFirer {
*
* @return the count of fired success events
*/
CompletableFuture<Integer> fireActiveEvents(IEventRepository eventRepository);
CompletableFuture<Integer> fireActiveEvents(IEventfulExecutionRunnable eventfulExecutionRunnable);
/**
* Shutdown the event firer, this method will block until all firing task has been completed.
*/
void shutdown();
}

13
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/IWorkflowEngine.java
Unescape View File

@ -21,10 +21,16 @@ import org.apache.dolphinscheduler.workflow.engine.exception.WorkflowExecuteRunn
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnable;
/**
* The WorkflowEngine is responsible for starting, stopping, pausing, and finalizing workflows.
* The WorkflowEngine is responsible for starting, stopping, pausing, and finalizing {@link IWorkflowExecutionRunnable}.
* All operation on a workflow instance should be done through the WorkflowEngine.
*/
public interface IWorkflowEngine {
/**
* Start the workflow engine.
*/
void start();
/**
* Trigger a workflow to start.
*
@ -55,4 +61,9 @@ public interface IWorkflowEngine {
*/
void finalizeWorkflow(Integer workflowInstanceId);
/**
* Shutdown the workflow engine. The workflow engine cannot be restarted after shutdown. This method will block until the workflow engine is completely shutdown.
*/
void shutdown();
}

21
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/WorkflowEngine.java
Unescape View File

@ -19,19 +19,28 @@ package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.workflow.engine.event.WorkflowOperationEvent;
import org.apache.dolphinscheduler.workflow.engine.exception.WorkflowExecuteRunnableNotFoundException;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecuteRunnableRepository;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionContext;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnableRepository;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class WorkflowEngine implements IWorkflowEngine {
private final IWorkflowExecuteRunnableRepository workflowExecuteRunnableRepository;
private final IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository;
public WorkflowEngine(IWorkflowExecuteRunnableRepository workflowExecuteRunnableRepository) {
private final IEventEngine eventEngine;
public WorkflowEngine(IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository,
IEventEngine eventEngine) {
this.workflowExecuteRunnableRepository = workflowExecuteRunnableRepository;
this.eventEngine = eventEngine;
}
@Override
public void start() {
eventEngine.start();
}
@Override
@ -74,9 +83,15 @@ public class WorkflowEngine implements IWorkflowEngine {
if (workflowExecutionRunnable == null) {
return;
}
// todo: If the workflowExecutionRunnable is not finished, we cannot finalize it.
log.info("Finalizing WorkflowExecutionRunnable: {}",
workflowExecutionRunnable.getWorkflowExecutionContext().getWorkflowInstanceName());
workflowExecuteRunnableRepository.removeWorkflowExecutionRunnable(workflowInstanceId);
}
@Override
public void shutdown() {
eventEngine.shutdown();
}
}

30
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/engine/WorkflowEngineFactory.java
Unescape View File

@ -17,12 +17,38 @@
package org.apache.dolphinscheduler.workflow.engine.engine;
import org.apache.dolphinscheduler.workflow.engine.workflow.SingletonWorkflowExecuteRunnableRepository;
import org.apache.dolphinscheduler.workflow.engine.workflow.IWorkflowExecutionRunnableRepository;
import org.apache.dolphinscheduler.workflow.engine.workflow.SingletonWorkflowExecutionRunnableRepository;
public class WorkflowEngineFactory implements IWorkflowEngineFactory {
private static final IWorkflowExecutionRunnableRepository DEFAULT_WORKFLOW_EXECUTION_RUNNABLE_FACTORY =
SingletonWorkflowExecutionRunnableRepository.getInstance();
private IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository =
DEFAULT_WORKFLOW_EXECUTION_RUNNABLE_FACTORY;
private IEventEngine eventEngine;
private WorkflowEngineFactory() {
}
public static WorkflowEngineFactory newWorkflowEngineFactory() {
return new WorkflowEngineFactory();
}
public WorkflowEngineFactory withWorkflowExecuteRunnableRepository(IWorkflowExecutionRunnableRepository workflowExecuteRunnableRepository) {
this.workflowExecuteRunnableRepository = workflowExecuteRunnableRepository;
return this;
}
public WorkflowEngineFactory withEventEngine(IEventEngine eventEngine) {
this.eventEngine = eventEngine;
return this;
}
@Override
public IWorkflowEngine createWorkflowEngine() {
return new WorkflowEngine(SingletonWorkflowExecuteRunnableRepository.getInstance());
return new WorkflowEngine(workflowExecuteRunnableRepository, eventEngine);
}
}

29
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/EventOperatorManager.java
Unescape View File

@ -17,20 +17,41 @@
package org.apache.dolphinscheduler.workflow.engine.event;
import lombok.extern.slf4j.Slf4j;
import java.util.HashMap;
import java.util.Map;
import org.springframework.stereotype.Component;
import lombok.extern.slf4j.Slf4j;
/**
* The event operator manager interface used to get {@link ITaskEventOperator}.
*/
@Slf4j
@Component
public class EventOperatorManager implements IEventOperatorManager<IEvent> {
private static final Map<String, IEventOperator<IEvent>> EVENT_OPERATOR_MAP = new HashMap<>();
private static final EventOperatorManager INSTANCE = new EventOperatorManager();
private EventOperatorManager() {
}
public static EventOperatorManager getInstance() {
return INSTANCE;
}
public void registerEventOperator(IEventOperator<IEvent> eventOperator) {
EVENT_OPERATOR_MAP.put(eventOperator.getClass().getSimpleName(), eventOperator);
}
@Override
public IEventOperator<IEvent> getEventOperator(IEvent event) {
return null;
if (event == null) {
throw new IllegalArgumentException("event cannot be null");
}
if (event.getEventOperatorClass() == null) {
throw new IllegalArgumentException("event operator class cannot be null");
}
return EVENT_OPERATOR_MAP.get(event.getEventOperatorClass().getSimpleName());
}
}

2
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/IEvent.java
Unescape View File

@ -19,4 +19,6 @@ package org.apache.dolphinscheduler.workflow.engine.event;
public interface IEvent {
Class getEventOperatorClass();
}

35
dolphinscheduler-workflow-engine/src/main/java/org/apache/dolphinscheduler/workflow/engine/event/IEventFirer.java
Unescape View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.workflow.engine.event;
import java.util.concurrent.CompletableFuture;
/**
* The event firer interface used to fire event.
*
*/
public interface IEventFirer {
/**
* Fire all active events in the event repository
*
* @return the count of fired success events
*/
CompletableFuture<Integer> fireActiveEvents(IEventRepository eventRepository);
}

Some files were not shown because too many files have changed in this diff Show More

Write Preview
Loading…
Cancel
Save