From c5b7e5adff40988f3dfd4f75dddb0b597509fff4 Mon Sep 17 00:00:00 2001 From: Tq Date: Tue, 19 Apr 2022 15:23:57 +0800 Subject: [PATCH] [Bug] [API-9558]fix homepage task instance count method to use submit time to recount (#9559) * fix homepage task instance count method to use submit time to recount * fix homepage task instance count method to use submit time to recount * fix homepage task instance count method to use submit time to recount * fix homepage task instance count method JUNIT * fix homepage task instance count method JUNIT * fix homepage task instance count method JUNIT --- .../api/service/DataAnalysisService.java | 33 ++++++++++--- .../service/impl/DataAnalysisServiceImpl.java | 49 ++++++++++++++----- .../dao/mapper/TaskInstanceMapper.java | 25 +++++++--- .../dao/mapper/TaskInstanceMapper.xml | 27 ++++++++++ .../task/api/enums/ExecutionStatus.java | 3 +- 5 files changed, 111 insertions(+), 26 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index 80810ed40e..cb4639acfc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -17,8 +17,12 @@ package org.apache.dolphinscheduler.api.service; +import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.ibatis.annotations.Param; +import java.util.Date; +import java.util.List; import java.util.Map; /** @@ -29,10 +33,10 @@ public interface DataAnalysisService { /** * statistical task instance status data * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code - * @param startDate start date - * @param endDate end date + * @param startDate start date + * @param endDate end date * @return task state count data */ Map countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate); @@ -40,20 +44,20 @@ public interface DataAnalysisService { /** * statistical process instance status data * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code - * @param startDate start date - * @param endDate end date + * @param startDate start date + * @param endDate end date * @return process instance state count data */ Map countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate); /** * statistics the process definition quantities of a certain person - * + *

* We only need projects which users have permission to see to determine whether the definition belongs to the user or not. * - * @param loginUser login user + * @param loginUser login user * @param projectCode project code * @return definition count data */ @@ -75,4 +79,17 @@ public interface DataAnalysisService { */ Map countQueueState(User loginUser); + /** + * Statistics task instance group by given project codes list + *

+ * We only need project codes to determine whether the task instance belongs to the user or not. + * + * @param startTime Statistics start time + * @param endTime Statistics end time + * @param projectCodes Project codes list to filter + * @return List of ExecuteStatusCount + */ + List countTaskInstanceAllStatesByProjectCodes(@Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("projectCodes") Long[] projectCodes); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java index 6617207a99..88449a49f0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.service.impl; import org.apache.dolphinscheduler.api.dto.CommandStateCount; import org.apache.dolphinscheduler.api.dto.DefineUserDto; import org.apache.dolphinscheduler.api.dto.TaskCountDto; +import org.apache.dolphinscheduler.api.dto.TaskStateCount; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.DataAnalysisService; import org.apache.dolphinscheduler.api.service.ProjectService; @@ -52,6 +53,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -101,11 +103,11 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal public Map countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate) { return countStateByProject( - loginUser, - projectCode, - startDate, - endDate, - (start, end, projectCodes) -> this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(start, end, projectCodes)); + loginUser, + projectCode, + startDate, + endDate, + this::countTaskInstanceAllStatesByProjectCodes); } /** @@ -119,15 +121,15 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal */ @Override public Map countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate) { - Map result = this.countStateByProject( + Map result = this.countStateByProject( loginUser, projectCode, startDate, endDate, - (start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes)); + (start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes)); // process state count needs to remove state of forced success if (result.containsKey(Constants.STATUS) && result.get(Constants.STATUS).equals(Status.SUCCESS)) { - ((TaskCountDto)result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS); + ((TaskCountDto) result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS); } return result; } @@ -163,9 +165,9 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal } } - List processInstanceStateCounts = new ArrayList<>(); Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser) - : new Long[] {projectCode}; + : new Long[]{projectCode}; + List processInstanceStateCounts = new ArrayList<>(); if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) { processInstanceStateCounts = instanceStateCounter.apply(start, end, projectCodeArray); @@ -203,7 +205,7 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal List defineGroupByUsers = new ArrayList<>(); Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser) - : new Long[] {projectCode}; + : new Long[]{projectCode}; if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) { defineGroupByUsers = processDefinitionMapper.countDefinitionByProjectCodes(projectCodeArray); } @@ -288,4 +290,29 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal return result; } + @Override + public List countTaskInstanceAllStatesByProjectCodes(Date startTime, Date endTime, Long[] projectCodes) { + Optional> startTimeStates = Optional.ofNullable(this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(startTime, endTime, projectCodes)); + + List allState = Arrays.stream(ExecutionStatus.values()).collect(Collectors.toList()); + List needRecountState; + if (startTimeStates.isPresent() && startTimeStates.get().size() != 0) { + List instanceState = startTimeStates.get().stream().map(ExecuteStatusCount::getExecutionStatus).collect(Collectors.toList()); + //value 0 state need to recount by submit time + needRecountState = allState.stream().filter(ele -> !instanceState.contains(ele)).collect(Collectors.toList()); + if (needRecountState.size() == 0) { + return startTimeStates.get(); + } + } else { + needRecountState = allState; + } + + //use submit time to recount when 0 + //if have any issues with this code, should change to specified states 0 8 9 17 not state count is 0 + List recounts = this.taskInstanceMapper + .countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(startTime, endTime, projectCodes, needRecountState); + startTimeStates.orElseGet(ArrayList::new).addAll(recounts); + + return startTimeStates.orElse(null); + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java index a79a07ca27..f98996952d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java @@ -17,19 +17,17 @@ package org.apache.dolphinscheduler.dao.mapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; - import org.apache.ibatis.annotations.Param; import java.util.Date; import java.util.List; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; - /** * task instance mapper interface */ @@ -58,7 +56,7 @@ public interface TaskInstanceMapper extends BaseMapper { @Param("taskIds") int[] taskIds); /** - * Statistics task instance group by given project codes list + * Statistics task instance group by given project codes list by start time *

* We only need project codes to determine whether the task instance belongs to the user or not. * @@ -71,6 +69,21 @@ public interface TaskInstanceMapper extends BaseMapper { @Param("endTime") Date endTime, @Param("projectCodes") Long[] projectCodes); + /** + * Statistics task instance group by given project codes list by submit time + *

+ * We only need project codes to determine whether the task instance belongs to the user or not. + * + * @param startTime Statistics start time + * @param endTime Statistics end time + * @param projectCodes Project codes list to filter + * @return List of ExecuteStatusCount + */ + List countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(@Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("projectCodes") Long[] projectCodes, + @Param("states") List states); + IPage queryTaskInstanceListPaging(IPage page, @Param("projectCode") Long projectCode, @Param("processInstanceId") Integer processInstanceId, @@ -84,5 +97,5 @@ public interface TaskInstanceMapper extends BaseMapper { @Param("endTime") Date endTime ); - List loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId,@Param("status") int status); + List loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId, @Param("status") int status); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index 8ea8459cd4..adc470c197 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -89,6 +89,33 @@ group by t.state +