[improve-#13045] after a submit failure, stop the processInstance to avoid an endless loop (#13051)

* [improve-#13045] add max submit number of workflow

* [fix-13405] remove max times

* Update dolphinscheduler-master/src/main/resources/application.yaml

Co-authored-by: Wenjun Ruan <wenjun@apache.org>

---------

Co-authored-by: fuchanghai <‘2875334588@qq.com’>
Co-authored-by: Wenjun Ruan <wenjun@apache.org>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java

@@ -27,7 +27,8 @@ public enum StateEventType {
     TASK_TIMEOUT(3, "task timeout"),
     WAKE_UP_TASK_GROUP(4, "wait task group"),
     TASK_RETRY(5, "task retry"),
-    PROCESS_BLOCKED(6, "process blocked");
+    PROCESS_BLOCKED(6, "process blocked"),
+    PROCESS_SUBMIT_FAILED(7, "process submit failed");
 
     StateEventType(int code, String descp) {
         this.code = code;

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java

@@ -17,6 +17,8 @@
 
 package org.apache.dolphinscheduler.server.master.event;
 
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
@@ -66,11 +68,16 @@ public class WorkflowStartEventHandler implements WorkflowEventHandler {
                         if (processInstance.getTimeout() > 0) {
                             stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
                         }
-                    } else {
-                        // submit failed will resend the event to workflow event queue
-                        log.error("Failed to submit the workflow instance, will resend the workflow start event: {}",
+                    } else if (WorkflowSubmitStatue.FAILED == workflowSubmitStatue) {
+                        log.error(
+                                "Failed to submit the workflow instance, will resend the workflow start event: {}",
                                 workflowEvent);
-                        workflowEventQueue.addEvent(workflowEvent);
+                        WorkflowStateEvent stateEvent = WorkflowStateEvent.builder()
+                                .processInstanceId(processInstance.getId())
+                                .type(StateEventType.PROCESS_SUBMIT_FAILED)
+                                .status(WorkflowExecutionStatus.FAILURE)
+                                .build();
+                        workflowExecuteRunnable.addStateEvent(stateEvent);
                     }
                 });
     }

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java

@@ -57,6 +57,9 @@ public class WorkflowStateEventHandler implements StateEventHandler {
             return true;
         }
         if (workflowStateEvent.getStatus().isFinished()) {
+            if (workflowStateEvent.getType().equals(StateEventType.PROCESS_SUBMIT_FAILED)) {
+                workflowExecuteRunnable.updateProcessInstanceState(workflowStateEvent);
+            }
             workflowExecuteRunnable.endProcess();
         }
         if (processInstance.getState().isReadyStop()) {