[Bug] [API-9558]fix homepage task instance count method to use submit time to recount (#9559)

* fix homepage task instance count method to use submit time to recount

* fix homepage task instance count method to use submit time to recount

* fix homepage task instance count method to use submit time to recount

* fix homepage task instance count method JUNIT

* fix homepage task instance count method JUNIT

* fix homepage task instance count method JUNIT

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java

@@ -17,8 +17,12 @@
 
 package org.apache.dolphinscheduler.api.service;
 
+import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
 import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.ibatis.annotations.Param;
 
+import java.util.Date;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -29,10 +33,10 @@ public interface DataAnalysisService {
     /**
      * statistical task instance status data
      *
-     * @param loginUser login user
+     * @param loginUser   login user
      * @param projectCode project code
-     * @param startDate start date
+     * @param startDate   start date
-     * @param endDate end date
+     * @param endDate     end date
      * @return task state count data
      */
     Map<String, Object> countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate);
@@ -40,20 +44,20 @@ public interface DataAnalysisService {
     /**
      * statistical process instance status data
      *
-     * @param loginUser login user
+     * @param loginUser   login user
      * @param projectCode project code
-     * @param startDate start date
+     * @param startDate   start date
-     * @param endDate end date
+     * @param endDate     end date
      * @return process instance state count data
      */
     Map<String, Object> countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate);
 
     /**
      * statistics the process definition quantities of a certain person
-     *
+     * <p>
      * We only need projects which users have permission to see to determine whether the definition belongs to the user or not.
      *
-     * @param loginUser login user
+     * @param loginUser   login user
      * @param projectCode project code
      * @return definition count data
      */
@@ -75,4 +79,17 @@ public interface DataAnalysisService {
      */
     Map<String, Object> countQueueState(User loginUser);
 
+    /**
+     * Statistics task instance group by given project codes list
+     * <p>
+     * We only need project codes to determine whether the task instance belongs to the user or not.
+     *
+     * @param startTime    Statistics start time
+     * @param endTime      Statistics end time
+     * @param projectCodes Project codes list to filter
+     * @return List of ExecuteStatusCount
+     */
+    List<ExecuteStatusCount> countTaskInstanceAllStatesByProjectCodes(@Param("startTime") Date startTime,
+                                                                      @Param("endTime") Date endTime,
+                                                                      @Param("projectCodes") Long[] projectCodes);
 }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java

@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.service.impl;
 import org.apache.dolphinscheduler.api.dto.CommandStateCount;
 import org.apache.dolphinscheduler.api.dto.DefineUserDto;
 import org.apache.dolphinscheduler.api.dto.TaskCountDto;
+import org.apache.dolphinscheduler.api.dto.TaskStateCount;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.service.DataAnalysisService;
 import org.apache.dolphinscheduler.api.service.ProjectService;
@@ -52,6 +53,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -101,11 +103,11 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
     public Map<String, Object> countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate) {
 
         return countStateByProject(
-            loginUser,
+                loginUser,
-            projectCode,
+                projectCode,
-            startDate,
+                startDate,
-            endDate,
+                endDate,
-            (start, end, projectCodes) -> this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(start, end, projectCodes));
+                this::countTaskInstanceAllStatesByProjectCodes);
     }
 
     /**
@@ -119,15 +121,15 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
      */
     @Override
     public Map<String, Object> countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate) {
-        Map<String, Object> result =  this.countStateByProject(
+        Map<String, Object> result = this.countStateByProject(
                 loginUser,
                 projectCode,
                 startDate,
                 endDate,
-            (start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes));
+                (start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes));
         // process state count needs to remove state of forced success
         if (result.containsKey(Constants.STATUS) && result.get(Constants.STATUS).equals(Status.SUCCESS)) {
-            ((TaskCountDto)result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS);
+            ((TaskCountDto) result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS);
         }
         return result;
     }
@@ -163,9 +165,9 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
             }
         }
 
-        List<ExecuteStatusCount> processInstanceStateCounts = new ArrayList<>();
         Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser)
-            : new Long[] {projectCode};
+                : new Long[]{projectCode};
+        List<ExecuteStatusCount> processInstanceStateCounts = new ArrayList<>();
 
         if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) {
             processInstanceStateCounts = instanceStateCounter.apply(start, end, projectCodeArray);
@@ -203,7 +205,7 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
 
         List<DefinitionGroupByUser> defineGroupByUsers = new ArrayList<>();
         Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser)
-            : new Long[] {projectCode};
+                : new Long[]{projectCode};
         if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) {
             defineGroupByUsers = processDefinitionMapper.countDefinitionByProjectCodes(projectCodeArray);
         }
@@ -288,4 +290,29 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
         return result;
     }
 
+    @Override
+    public List<ExecuteStatusCount> countTaskInstanceAllStatesByProjectCodes(Date startTime, Date endTime, Long[] projectCodes) {
+        Optional<List<ExecuteStatusCount>> startTimeStates = Optional.ofNullable(this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(startTime, endTime, projectCodes));
+
+        List<ExecutionStatus> allState = Arrays.stream(ExecutionStatus.values()).collect(Collectors.toList());
+        List<ExecutionStatus> needRecountState;
+        if (startTimeStates.isPresent() && startTimeStates.get().size() != 0) {
+            List<ExecutionStatus> instanceState = startTimeStates.get().stream().map(ExecuteStatusCount::getExecutionStatus).collect(Collectors.toList());
+            //value 0 state need to recount by submit time
+            needRecountState = allState.stream().filter(ele -> !instanceState.contains(ele)).collect(Collectors.toList());
+            if (needRecountState.size() == 0) {
+                return startTimeStates.get();
+            }
+        } else {
+            needRecountState = allState;
+        }
+
+        //use submit time to recount when 0
+        //if have any issues with this code, should change to specified states 0 8 9 17 not state count is 0
+        List<ExecuteStatusCount> recounts = this.taskInstanceMapper
+                .countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(startTime, endTime, projectCodes, needRecountState);
+        startTimeStates.orElseGet(ArrayList::new).addAll(recounts);
+
+        return startTimeStates.orElse(null);
+    }
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java

@@ -17,19 +17,17 @@
 
 package org.apache.dolphinscheduler.dao.mapper;
 
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
 import org.apache.dolphinscheduler.common.enums.Flag;
 import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-
 import org.apache.ibatis.annotations.Param;
 
 import java.util.Date;
 import java.util.List;
 
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-
 /**
  * task instance mapper interface
  */
@@ -58,7 +56,7 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
                       @Param("taskIds") int[] taskIds);
 
     /**
-     * Statistics task instance group by given project codes list
+     * Statistics task instance group by given project codes list by start time
      * <p>
      * We only need project codes to determine whether the task instance belongs to the user or not.
      *
@@ -71,6 +69,21 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
                                                                   @Param("endTime") Date endTime,
                                                                   @Param("projectCodes") Long[] projectCodes);
 
+    /**
+     * Statistics task instance group by given project codes list by submit time
+     * <p>
+     * We only need project codes to determine whether the task instance belongs to the user or not.
+     *
+     * @param startTime    Statistics start time
+     * @param endTime      Statistics end time
+     * @param projectCodes Project codes list to filter
+     * @return List of ExecuteStatusCount
+     */
+    List<ExecuteStatusCount> countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(@Param("startTime") Date startTime,
+                                                                                       @Param("endTime") Date endTime,
+                                                                                       @Param("projectCodes") Long[] projectCodes,
+                                                                                       @Param("states") List<ExecutionStatus> states);
+
     IPage<TaskInstance> queryTaskInstanceListPaging(IPage<TaskInstance> page,
                                                     @Param("projectCode") Long projectCode,
                                                     @Param("processInstanceId") Integer processInstanceId,
@@ -84,5 +97,5 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
                                                     @Param("endTime") Date endTime
     );
 
-    List<TaskInstance> loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId,@Param("status") int status);
+    List<TaskInstance> loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId, @Param("status") int status);
 }

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml

@@ -89,6 +89,33 @@
         </if>
         group by t.state
     </select>
+    <select id="countTaskInstanceStateByProjectCodesAndStatesBySubmitTime" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
+        select
+        state, count(0) as count
+        from t_ds_task_instance t
+        left join t_ds_task_definition_log d on d.code=t.task_code and d.version=t.task_definition_version
+        left join t_ds_project p on p.code=d.project_code
+        where 1=1
+        <if test="states != null and states.size != 0">
+            and t.state in
+            <foreach collection="states" index="index" item="state" open="(" separator="," close=")">
+                #{state.code}
+            </foreach>
+        </if>
+        <if test="projectCodes != null and projectCodes.length != 0">
+            and d.project_code in
+            <foreach collection="projectCodes" index="index" item="i" open="(" separator="," close=")">
+                #{i}
+            </foreach>
+        </if>
+        <if test="startTime != null">
+            and t.submit_time <![CDATA[ > ]]> #{startTime}
+        </if>
+        <if test="endTime != null">
+            and t.submit_time <![CDATA[ <= ]]> #{endTime}
+        </if>
+        group by t.state
+    </select>
     <select id="queryByInstanceIdAndName" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
         select
         <include refid="baseSql"/>

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java

@@ -45,6 +45,7 @@ public enum ExecutionStatus {
      * 14 serial wait
      * 15 ready block
      * 16 block
+     * 17 dispatch
      */
     SUBMITTED_SUCCESS(0, "submit success"),
     RUNNING_EXECUTION(1, "running"),
@@ -108,7 +109,7 @@ public enum ExecutionStatus {
      */
     public boolean typeIsFinished() {
         return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause()
-            || typeIsStop() || typeIsBlock();
+                || typeIsStop() || typeIsBlock();
     }
 
     /**