[Feature][JsonSplit] Fix api bug (#5102)

* modify checkDAGRing and ProcessService method

* merge

* modify dagRing

* modify process instance for project home page

* fix save process bug

* codeStyle

* Fix logical bug in saving process definition

* codeSytle

* Fix bug in interface of  queryProcessDefinitionList

* codeSytle

* Fix api bug"

Co-authored-by: JinyLeeChina <297062848@qq.com>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java

@@ -43,13 +43,14 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
 import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -148,8 +149,8 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
             }
         }
 
-        Project project = projectMapper.selectById(projectId);
+        Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
-        Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode());
+                : new Long[] { projectMapper.selectById(projectId).getCode() };
         List<ExecuteStatusCount> processInstanceStateCounts =
                 instanceStateCounter.apply(start, end, projectCodeArray);
 
@@ -172,9 +173,12 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
     @Override
     public Map<String, Object> countDefinitionByUser(User loginUser, int projectId) {
         Map<String, Object> result = new HashMap<>();
-
+        boolean checkProject = checkProject(loginUser, projectId, result);
-        Project project = projectMapper.selectById(projectId);
+        if (!checkProject) {
-        Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode());
+            return result;
+        }
+        Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
+                : new Long[] { projectMapper.selectById(projectId).getCode() };
         List<DefinitionGroupByUser> defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser(
                 loginUser.getId(), projectCodeArray, isAdmin(loginUser));
 
@@ -203,8 +207,6 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
             return result;
         }
 
-        Project project = projectMapper.selectById(projectId);
-
         /**
          * find all the task lists in the project under the user
          * statistics based on task status execution, failure, completion, wait, total
@@ -226,7 +228,8 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
             }
         }
 
-        Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode());
+        Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
+                : new Long[] { projectMapper.selectById(projectId).getCode() };
         // count normal command state
         Map<CommandType, Integer> normalCountCommandCounts = commandMapper.countCommandState(loginUser.getId(), start, end, projectCodeArray)
                 .stream()
@@ -249,15 +252,14 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
         return result;
     }
 
-    private Long[] getProjectCodesArrays(User loginUser, Long projectCode) {
+    private Long[] getProjectCodesArrays(User loginUser) {
-        List<Long> projectCodes = new ArrayList<>();
+        List<Project> projectList = projectMapper.queryRelationProjectListByUserId(
-        if (projectCode != 0) {
+                loginUser.getUserType() == UserType.ADMIN_USER ? 0 : loginUser.getId());
-            projectCodes.add(projectCode);
+        Set<Long> projectCodes = new HashSet<>();
-        } else if (loginUser.getUserType() == UserType.GENERAL_USER) {
+        projectList.forEach(project -> projectCodes.add(project.getCode()));
-            projectCodes = processService.getProjectIdListHavePerm(loginUser.getId());
+        if (loginUser.getUserType() == UserType.GENERAL_USER) {
-            if (projectCodes.isEmpty()) {
+            List<Project> createProjects = projectMapper.queryProjectCreatedByUser(loginUser.getId());
-                projectCodes.add(0L);
+            createProjects.forEach(project -> projectCodes.add(project.getCode()));
-            }
         }
         return projectCodes.toArray(new Long[0]);
     }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@@ -259,9 +259,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
 
         ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefineId);
 
-        IPage<ProcessInstance> processInstanceList =
+        IPage<ProcessInstance> processInstanceList = processInstanceMapper.queryProcessInstanceListPaging(page,
-                processInstanceMapper.queryProcessInstanceListPaging(page,
+                        project.getCode(), processDefinition == null ? 0L : processDefinition.getCode(), searchVal,
-                        project.getCode(), processDefinition.getCode(), searchVal, executorId, statusArray, host, start, end);
+                        executorId, statusArray, host, start, end);
 
         List<ProcessInstance> processInstances = processInstanceList.getRecords();
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java

@@ -444,7 +444,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
             return Constants.ALL_PERMISSIONS;
         }
 
-        ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getCode(), user.getId());
+        ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getId(), user.getId());
 
         if (projectUser == null) {
             return 0;

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

@@ -271,25 +271,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
             return result;
         }
         TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskCode, version);
-        taskDefinition.setVersion(version);
+        taskDefinitionLog.setUserId(loginUser.getId());
-        taskDefinition.setCode(taskCode);
+        taskDefinitionLog.setUpdateTime(new Date());
-        taskDefinition.setName(taskDefinitionLog.getName());
+        taskDefinitionMapper.updateById(taskDefinitionLog);
-        taskDefinition.setDescription(taskDefinitionLog.getDescription());
-        taskDefinition.setProjectCode(taskDefinitionLog.getProjectCode());
-        taskDefinition.setUserId(loginUser.getId());
-        taskDefinition.setTaskType(taskDefinitionLog.getTaskType());
-        taskDefinition.setTaskParams(taskDefinitionLog.getTaskParams());
-        taskDefinition.setFlag(taskDefinitionLog.getFlag());
-        taskDefinition.setTaskPriority(taskDefinitionLog.getTaskPriority());
-        taskDefinition.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
-        taskDefinition.setFailRetryTimes(taskDefinitionLog.getFailRetryTimes());
-        taskDefinition.setFailRetryInterval(taskDefinitionLog.getFailRetryInterval());
-        taskDefinition.setTimeoutFlag(taskDefinitionLog.getTimeoutFlag());
-        taskDefinition.setTimeoutNotifyStrategy(taskDefinitionLog.getTimeoutNotifyStrategy());
-        taskDefinition.setTimeout(taskDefinitionLog.getTimeout());
-        taskDefinition.setUpdateTime(new Date());
-        taskDefinition.setResourceIds(taskDefinitionLog.getResourceIds());
-        taskDefinitionMapper.updateById(taskDefinition);
         result.put(Constants.DATA_LIST, taskCode);
         putMsg(result, Status.SUCCESS);
         return result;

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@@ -525,10 +525,10 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
             putMsg(result, Status.USER_NOT_EXIST, userId);
             return result;
         }
-        //if the selected projectIds are empty, delete all items associated with the user
-        projectUserMapper.deleteProjectRelation(0L, userId);
 
+        //if the selected projectIds are empty, delete all items associated with the user
         if (check(result, StringUtils.isEmpty(projectIds), Status.SUCCESS)) {
+            projectUserMapper.deleteProjectRelation(0, userId);
             return result;
         }
 

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java

@@ -116,7 +116,7 @@ public class ProjectServiceTest {
     @Test
     public void testCheckProjectAndAuth() {
 
-        Mockito.when(projectUserMapper.queryProjectRelation(1L, 1)).thenReturn(getProjectUser());
+        Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
         User loginUser = getLoginUser();
 
         Map<String, Object> result = projectService.checkProjectAndAuth(loginUser, null, projectName);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@@ -305,7 +305,7 @@ public class UsersServiceTest {
         logger.info(result.toString());
         Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
         //success
-        when(projectUserMapper.deleteProjectRelation(Mockito.anyLong(), Mockito.anyInt())).thenReturn(1);
+        when(projectUserMapper.deleteProjectRelation(Mockito.anyInt(), Mockito.anyInt())).thenReturn(1);
         result = usersService.grantProject(loginUser, 1, projectIds);
         logger.info(result.toString());
         Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java

@@ -137,12 +137,6 @@ public class TaskNode {
      */
     private String workerGroup;
 
-    /**
-     * worker group id
-     */
-    private Integer workerGroupId;
-
-
     /**
      * task time out
      */
@@ -325,14 +319,6 @@ public class TaskNode {
         this.conditionResult = conditionResult;
     }
 
-    public Integer getWorkerGroupId() {
-        return workerGroupId;
-    }
-
-    public void setWorkerGroupId(Integer workerGroupId) {
-        this.workerGroupId = workerGroupId;
-    }
-
     public int getDelayTime() {
         return delayTime;
     }
@@ -405,7 +391,6 @@ public class TaskNode {
                 + ", conditionResult='" + conditionResult + '\''
                 + ", taskInstancePriority=" + taskInstancePriority
                 + ", workerGroup='" + workerGroup + '\''
-                + ", workerGroupId=" + workerGroupId
                 + ", timeout='" + timeout + '\''
                 + ", delayTime=" + delayTime
                 + '}';

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectUser.java

@@ -34,12 +34,9 @@ public class ProjectUser {
     @TableField("user_id")
     private int userId;
 
-    @TableField(exist = false)
+    @TableField("project_id")
     private int projectId;
 
-    @TableField("project_code")
-    private long projectCode;
-
     /**
      * project name
      */
@@ -63,14 +60,6 @@ public class ProjectUser {
     @TableField("update_time")
     private Date updateTime;
 
-    public long getProjectCode() {
-        return projectCode;
-    }
-
-    public void setProjectCode(long projectCode) {
-        this.projectCode = projectCode;
-    }
-
     public int getId() {
         return id;
     }
@@ -141,7 +130,6 @@ public class ProjectUser {
                + "id=" + id
                + ", userId=" + userId
                + ", projectId=" + projectId
-               + ", projectCode=" + projectCode
                + ", projectName='" + projectName + '\''
                + ", userName='" + userName + '\''
                + ", perm=" + perm

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java

@@ -85,6 +85,13 @@ public interface ProjectMapper extends BaseMapper<Project> {
      */
     List<Project> queryAuthedProjectListByUserId(@Param("userId") int userId);
 
+    /**
+     * query relation project list by userId
+     * @param userId userId
+     * @return project list
+     */
+    List<Project> queryRelationProjectListByUserId(@Param("userId") int userId);
+
     /**
      * query project except userId
      * @param userId userId

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.java

@@ -17,9 +17,11 @@
 package org.apache.dolphinscheduler.dao.mapper;
 
 import org.apache.dolphinscheduler.dao.entity.ProjectUser;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
 import org.apache.ibatis.annotations.Param;
 
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
 /**
  * project user mapper interface
  */
@@ -27,19 +29,21 @@ public interface ProjectUserMapper extends BaseMapper<ProjectUser> {
 
     /**
      * delte prject user relation
-     * @param projectCode projectCode
+     *
+     * @param projectId projectId
      * @param userId userId
      * @return delete result
      */
-    int deleteProjectRelation(@Param("projectCode") Long projectCode,
+    int deleteProjectRelation(@Param("projectId") int projectId,
                               @Param("userId") int userId);
 
     /**
      * query project relation
-     * @param projectCode projectCode
+     *
+     * @param projectId projectId
      * @param userId userId
      * @return project user relation
      */
-    ProjectUser queryProjectRelation(@Param("projectCode") Long projectCode,
+    ProjectUser queryProjectRelation(@Param("projectId") int projectId,
                                      @Param("userId") int userId);
 }

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.xml

@@ -29,7 +29,7 @@
             </foreach>
         </if>
         <if test="startTime != null and endTime != null">
-            and cmd.startTime <![CDATA[ >= ]]> #{startTime} and cmd.update_time <![CDATA[ <= ]]> #{endTime}
+            and cmd.start_time <![CDATA[ >= ]]> #{startTime} and cmd.update_time <![CDATA[ <= ]]> #{endTime}
         </if>
         group by cmd.command_type
     </select>

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml

@@ -19,7 +19,6 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
 <mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectMapper">
     <sql id="baseSql">
-
         id, name, code, description, user_id, flag, create_time, update_time
     </sql>
     <sql id="baseSqlV2">
@@ -83,9 +82,9 @@
         join t_ds_user u on u.id=p.user_id
         where 1=1
         <if test="userId != 0">
-            and p.code in
+            and p.id in
-            (select project_code from t_ds_relation_project_user where user_id=#{userId}
+            (select project_id from t_ds_relation_project_user where user_id=#{userId}
-            union select id as project_code from t_ds_project where user_id=#{userId}
+            union select id as project_id from t_ds_project where user_id=#{userId}
             )
         </if>
         <if test="searchName!=null and searchName != ''">
@@ -101,6 +100,18 @@
         from t_ds_project p,t_ds_relation_project_user rel
         where p.id = rel.project_id and rel.user_id= #{userId}
     </select>
+    <select id="queryRelationProjectListByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
+        select
+        <include refid="baseSqlV2">
+            <property name="alias" value="p"/>
+        </include>
+        from t_ds_project p left join t_ds_relation_project_user rel
+        on p.id = rel.project_id
+        where 1=1
+        <if test="userId != 0 ">
+            and rel.user_id= #{userId}
+        </if>
+    </select>
     <select id="queryProjectExceptUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
         select
         <include refid="baseSql"/>
@@ -114,16 +125,18 @@
         where user_id = #{userId}
     </select>
     <select id="queryProjectCreatedAndAuthorizedByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
-        select * from t_ds_project where id in
+        select
+        <include refid="baseSql"/>
+        from t_ds_project where id in
            (select project_id from t_ds_relation_project_user  where user_id=#{userId}
              union select id as project_id  from t_ds_project where user_id=#{userId})
     </select>
 
     <select id="queryProjectWithUserByProcessInstanceId" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser">
         select
-            dp.code projectCode,
+            dp.id project_id,
-            dp.name projectName,
+            dp.name project_name,
-            u.user_name userName
+            u.user_name user_name
         from t_ds_process_instance di
         join t_ds_process_definition dpd on di.process_definition_code = dpd.code
         join t_ds_project dp on dpd.project_code = dp.code

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.xml

@@ -19,21 +19,21 @@
 <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
 <mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper">
     <sql id="baseSql">
-        id, user_id, project_code, perm, create_time, update_time
+        id, user_id, project_id, perm, create_time, update_time
     </sql>
     <delete id="deleteProjectRelation">
         delete from t_ds_relation_project_user
         where 1=1
         and user_id = #{userId}
-        <if test="projectCode != 0 ">
+        <if test="projectId != 0 ">
-            and project_code = #{projectCode}
+            and project_id = #{projectId}
         </if>
     </delete>
     <select id="queryProjectRelation" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser">
         select
         <include refid="baseSql"/>
         from t_ds_relation_project_user
-        where project_code = #{projectCode}
+        where project_id = #{projectId}
         and user_id = #{userId}
         limit 1
     </select>

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java

@@ -52,7 +52,6 @@ public class ProjectUserMapperTest {
     private ProjectUser insertOne(){
         //insertOne
         ProjectUser projectUser = new ProjectUser();
-        projectUser.setProjectCode(1010L);
         projectUser.setProjectId(1010);
         projectUser.setUserId(111);
         projectUserMapper.insert(projectUser);
@@ -101,7 +100,7 @@ public class ProjectUserMapperTest {
 
 
         ProjectUser projectUser = insertOne();
-        int delete = projectUserMapper.deleteProjectRelation(projectUser.getProjectCode(), projectUser.getUserId());
+        int delete = projectUserMapper.deleteProjectRelation(projectUser.getProjectId(), projectUser.getUserId());
         assertThat(delete,greaterThanOrEqualTo(1));
 
     }
@@ -112,7 +111,7 @@ public class ProjectUserMapperTest {
     @Test
     public void testQueryProjectRelation() {
         ProjectUser projectUser = insertOne();
-        ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectCode(), projectUser.getUserId());
+        ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId());
         Assert.assertNotEquals(projectUser1, null);
 
     }

dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@@ -2222,7 +2222,6 @@ public class ProcessService {
      * update task definition
      */
     public int updateTaskDefinition(User operator, Long projectCode, TaskNode taskNode, TaskDefinition taskDefinition) {
-
         Integer version = taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode());
         Date now = new Date();
         taskDefinition.setProjectCode(projectCode);
@@ -2243,7 +2242,7 @@ public class ProcessService {
         taskDefinition.setName(taskNode.getName());
         taskDefinition.setDescription(taskNode.getDesc());
         taskDefinition.setTaskType(TaskType.of(taskNode.getType()));
-        taskDefinition.setTaskParams(taskNode.getParams());
+        taskDefinition.setTaskParams(TaskType.of(taskNode.getType()) == TaskType.DEPENDENT ? taskNode.getDependence() : taskNode.getParams());
         taskDefinition.setFlag(taskNode.isForbidden() ? Flag.NO : Flag.YES);
         taskDefinition.setTaskPriority(taskNode.getTaskInstancePriority());
         taskDefinition.setWorkerGroup(taskNode.getWorkerGroup());
@@ -2508,7 +2507,8 @@ public class ProcessService {
             v.setRunFlag(taskDefinitionLog.getFlag() == Flag.YES ? Constants.FLOWNODE_RUN_FLAG_NORMAL : Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
             v.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes());
             v.setRetryInterval(taskDefinitionLog.getFailRetryInterval());
-            v.setParams(taskDefinitionLog.getTaskParams());
+            v.setParams(taskDefinitionLog.getTaskType() == TaskType.DEPENDENT ? "" : taskDefinitionLog.getTaskParams());
+            v.setDependence(taskDefinitionLog.getTaskType() == TaskType.DEPENDENT ? taskDefinitionLog.getTaskParams() : "");
             v.setTaskInstancePriority(taskDefinitionLog.getTaskPriority());
             v.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
             v.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN,

sql/dolphinscheduler_mysql.sql

@@ -696,7 +696,7 @@ DROP TABLE IF EXISTS `t_ds_relation_project_user`;
 CREATE TABLE `t_ds_relation_project_user` (
   `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
   `user_id` int(11) NOT NULL COMMENT 'user id',
-  `project_code` bigint(20) DEFAULT NULL COMMENT 'project code',
+  `project_id` int(11) DEFAULT NULL COMMENT 'project id',
   `perm` int(11) DEFAULT '1' COMMENT 'limits of authority',
   `create_time` datetime DEFAULT NULL COMMENT 'create time',
   `update_time` datetime DEFAULT NULL COMMENT 'update time',