Browse Source

Merge branch 'json_split' of https://github.com/apache/incubator-dolphinscheduler into spilit

pull/3/MERGE
lenboo 4 years ago
parent
commit
0914699fc5
  1. 38
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
  2. 28
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  3. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
  4. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
  5. 22
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
  6. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  7. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
  8. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
  9. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java
  10. 15
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
  11. 30
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  12. 14
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectUser.java
  13. 5
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java
  14. 7
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java
  15. 14
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.java
  16. 66
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
  17. 2
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.xml
  18. 8
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.xml
  19. 29
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
  20. 8
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.xml
  21. 23
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapperTest.java
  22. 5
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java
  23. 15
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  24. 179
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
  25. 18
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue
  26. 2
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/index.vue
  27. 2
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue
  28. 2
      sql/dolphinscheduler_mysql.sql

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java

@ -43,13 +43,14 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -148,8 +149,8 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
} }
} }
Project project = projectMapper.selectById(projectId); Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode()); : new Long[] { projectMapper.selectById(projectId).getCode() };
List<ExecuteStatusCount> processInstanceStateCounts = List<ExecuteStatusCount> processInstanceStateCounts =
instanceStateCounter.apply(start, end, projectCodeArray); instanceStateCounter.apply(start, end, projectCodeArray);
@ -172,9 +173,12 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
@Override @Override
public Map<String, Object> countDefinitionByUser(User loginUser, int projectId) { public Map<String, Object> countDefinitionByUser(User loginUser, int projectId) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
boolean checkProject = checkProject(loginUser, projectId, result);
Project project = projectMapper.selectById(projectId); if (!checkProject) {
Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode()); return result;
}
Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectMapper.selectById(projectId).getCode() };
List<DefinitionGroupByUser> defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser( List<DefinitionGroupByUser> defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser(
loginUser.getId(), projectCodeArray, isAdmin(loginUser)); loginUser.getId(), projectCodeArray, isAdmin(loginUser));
@ -203,8 +207,6 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
return result; return result;
} }
Project project = projectMapper.selectById(projectId);
/** /**
* find all the task lists in the project under the user * find all the task lists in the project under the user
* statistics based on task status execution, failure, completion, wait, total * statistics based on task status execution, failure, completion, wait, total
@ -226,7 +228,8 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
} }
} }
Long[] projectCodeArray = getProjectCodesArrays(loginUser, project.getCode()); Long[] projectCodeArray = projectId == 0 ? getProjectCodesArrays(loginUser)
: new Long[] { projectMapper.selectById(projectId).getCode() };
// count normal command state // count normal command state
Map<CommandType, Integer> normalCountCommandCounts = commandMapper.countCommandState(loginUser.getId(), start, end, projectCodeArray) Map<CommandType, Integer> normalCountCommandCounts = commandMapper.countCommandState(loginUser.getId(), start, end, projectCodeArray)
.stream() .stream()
@ -249,15 +252,14 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
return result; return result;
} }
private Long[] getProjectCodesArrays(User loginUser, Long projectCode) { private Long[] getProjectCodesArrays(User loginUser) {
List<Long> projectCodes = new ArrayList<>(); List<Project> projectList = projectMapper.queryRelationProjectListByUserId(
if (projectCode != 0) { loginUser.getUserType() == UserType.ADMIN_USER ? 0 : loginUser.getId());
projectCodes.add(projectCode); Set<Long> projectCodes = new HashSet<>();
} else if (loginUser.getUserType() == UserType.GENERAL_USER) { projectList.forEach(project -> projectCodes.add(project.getCode()));
projectCodes = processService.getProjectIdListHavePerm(loginUser.getId()); if (loginUser.getUserType() == UserType.GENERAL_USER) {
if (projectCodes.isEmpty()) { List<Project> createProjects = projectMapper.queryProjectCreatedByUser(loginUser.getId());
projectCodes.add(0L); createProjects.forEach(project -> projectCodes.add(project.getCode()));
}
} }
return projectCodes.toArray(new Long[0]); return projectCodes.toArray(new Long[0]);
} }

28
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -66,6 +66,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.permission.PermissionCheck; import org.apache.dolphinscheduler.service.permission.PermissionCheck;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
@ -123,6 +124,9 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
@Autowired @Autowired
private ProjectService projectService; private ProjectService projectService;
@Autowired
private UserMapper userMapper;
@Autowired @Autowired
private ProcessDefinitionLogMapper processDefinitionLogMapper; private ProcessDefinitionLogMapper processDefinitionLogMapper;
@ -270,11 +274,21 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
IPage<ProcessDefinition> processDefinitionIPage = processDefinitionMapper.queryDefineListPaging( IPage<ProcessDefinition> processDefinitionIPage = processDefinitionMapper.queryDefineListPaging(
page, searchVal, userId, project.getCode(), isAdmin(loginUser)); page, searchVal, userId, project.getCode(), isAdmin(loginUser));
processDefinitionIPage.setRecords(processDefinitionIPage.getRecords()); List<ProcessDefinition> records = processDefinitionIPage.getRecords();
for (ProcessDefinition pd : records) {
ProcessDefinitionLog processDefinitionLog = processDefinitionLogMapper.queryMaxVersionDefinitionLog(pd.getCode());
int operator = processDefinitionLog.getOperator();
User user = userMapper.selectById(operator);
pd.setModifyBy(user.getUserName());
pd.setProjectId(project.getId());
}
processDefinitionIPage.setRecords(records);
PageInfo<ProcessDefinition> pageInfo = new PageInfo<>(pageNo, pageSize); PageInfo<ProcessDefinition> pageInfo = new PageInfo<>(pageNo, pageSize);
pageInfo.setTotalCount((int) processDefinitionIPage.getTotal()); pageInfo.setTotalCount((int) processDefinitionIPage.getTotal());
pageInfo.setLists(processDefinitionIPage.getRecords()); pageInfo.setLists(records);
result.put(Constants.DATA_LIST, pageInfo); result.put(Constants.DATA_LIST, pageInfo);
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
@ -1384,7 +1398,10 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
} else { } else {
ProcessData processData = processService.genProcessData(processDefinition); ProcessData processData = processService.genProcessData(processDefinition);
List<TaskNode> taskNodeList = processData.getTasks(); List<TaskNode> taskNodeList = processData.getTasks();
taskNodeList.forEach(taskNode -> taskNode.setCode(0L)); taskNodeList.forEach(taskNode -> {
taskNode.setName(taskNode.getName() + "_copy_" + DateUtils.getCurrentTimeStamp());
taskNode.setCode(0L);
});
processData.setTasks(taskNodeList); processData.setTasks(taskNodeList);
String processDefinitionJson = JSONUtils.toJsonString(processData); String processDefinitionJson = JSONUtils.toJsonString(processData);
return createProcessDefinition( return createProcessDefinition(
@ -1566,6 +1583,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
} }
} catch (Exception e) { } catch (Exception e) {
setFailedProcessList(failedProcessList, processDefinitionId); setFailedProcessList(failedProcessList, processDefinitionId);
logger.error("move processDefinition error: {}", e.getMessage(), e);
} }
} }
} }
@ -1589,6 +1608,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
} }
} catch (Exception e) { } catch (Exception e) {
setFailedProcessList(failedProcessList, processDefinitionId); setFailedProcessList(failedProcessList, processDefinitionId);
logger.error("copy processDefinition error: {}", e.getMessage(), e);
} }
} }
} }
@ -1676,6 +1697,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
} }
} }
/** /**
* check has associated process definition * check has associated process definition
* *

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@ -259,9 +259,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefineId); ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefineId);
IPage<ProcessInstance> processInstanceList = IPage<ProcessInstance> processInstanceList = processInstanceMapper.queryProcessInstanceListPaging(page,
processInstanceMapper.queryProcessInstanceListPaging(page, project.getCode(), processDefinition == null ? 0L : processDefinition.getCode(), searchVal,
project.getCode(), processDefinition.getCode(), searchVal, executorId, statusArray, host, start, end); executorId, statusArray, host, start, end);
List<ProcessInstance> processInstances = processInstanceList.getRecords(); List<ProcessInstance> processInstances = processInstanceList.getRecords();
List<Integer> userIds = CollectionUtils.transformToList(processInstances, ProcessInstance::getExecutorId); List<Integer> userIds = CollectionUtils.transformToList(processInstances, ProcessInstance::getExecutorId);

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java

@ -444,7 +444,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
return Constants.ALL_PERMISSIONS; return Constants.ALL_PERMISSIONS;
} }
ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getCode(), user.getId()); ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getId(), user.getId());
if (projectUser == null) { if (projectUser == null) {
return 0; return 0;

22
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

@ -271,25 +271,9 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
return result; return result;
} }
TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskCode, version); TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMapper.queryByDefinitionCodeAndVersion(taskCode, version);
taskDefinition.setVersion(version); taskDefinitionLog.setUserId(loginUser.getId());
taskDefinition.setCode(taskCode); taskDefinitionLog.setUpdateTime(new Date());
taskDefinition.setName(taskDefinitionLog.getName()); taskDefinitionMapper.updateById(taskDefinitionLog);
taskDefinition.setDescription(taskDefinitionLog.getDescription());
taskDefinition.setProjectCode(taskDefinitionLog.getProjectCode());
taskDefinition.setUserId(loginUser.getId());
taskDefinition.setTaskType(taskDefinitionLog.getTaskType());
taskDefinition.setTaskParams(taskDefinitionLog.getTaskParams());
taskDefinition.setFlag(taskDefinitionLog.getFlag());
taskDefinition.setTaskPriority(taskDefinitionLog.getTaskPriority());
taskDefinition.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
taskDefinition.setFailRetryTimes(taskDefinitionLog.getFailRetryTimes());
taskDefinition.setFailRetryInterval(taskDefinitionLog.getFailRetryInterval());
taskDefinition.setTimeoutFlag(taskDefinitionLog.getTimeoutFlag());
taskDefinition.setTimeoutNotifyStrategy(taskDefinitionLog.getTimeoutNotifyStrategy());
taskDefinition.setTimeout(taskDefinitionLog.getTimeout());
taskDefinition.setUpdateTime(new Date());
taskDefinition.setResourceIds(taskDefinitionLog.getResourceIds());
taskDefinitionMapper.updateById(taskDefinition);
result.put(Constants.DATA_LIST, taskCode); result.put(Constants.DATA_LIST, taskCode);
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result; return result;

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -533,10 +533,10 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
putMsg(result, Status.USER_NOT_EXIST, userId); putMsg(result, Status.USER_NOT_EXIST, userId);
return result; return result;
} }
//if the selected projectIds are empty, delete all items associated with the user
projectUserMapper.deleteProjectRelation(0L, userId);
//if the selected projectIds are empty, delete all items associated with the user
if (check(result, StringUtils.isEmpty(projectIds), Status.SUCCESS)) { if (check(result, StringUtils.isEmpty(projectIds), Status.SUCCESS)) {
projectUserMapper.deleteProjectRelation(0, userId);
return result; return result;
} }

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java

@ -116,7 +116,7 @@ public class ProjectServiceTest {
@Test @Test
public void testCheckProjectAndAuth() { public void testCheckProjectAndAuth() {
Mockito.when(projectUserMapper.queryProjectRelation(1L, 1)).thenReturn(getProjectUser()); Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser());
User loginUser = getLoginUser(); User loginUser = getLoginUser();
Map<String, Object> result = projectService.checkProjectAndAuth(loginUser, null, projectName); Map<String, Object> result = projectService.checkProjectAndAuth(loginUser, null, projectName);

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@ -318,7 +318,7 @@ public class UsersServiceTest {
logger.info(result.toString()); logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success //success
when(projectUserMapper.deleteProjectRelation(Mockito.anyLong(), Mockito.anyInt())).thenReturn(1); when(projectUserMapper.deleteProjectRelation(Mockito.anyInt(), Mockito.anyInt())).thenReturn(1);
result = usersService.grantProject(loginUser, 1, projectIds); result = usersService.grantProject(loginUser, 1, projectIds);
logger.info(result.toString()); logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java

@ -34,6 +34,10 @@ public class PreviousTaskNode {
*/ */
private int version; private int version;
public PreviousTaskNode() {
}
public PreviousTaskNode(long code, String name, int version) { public PreviousTaskNode(long code, String name, int version) {
this.code = code; this.code = code;
this.name = name; this.name = name;

15
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java

@ -137,12 +137,6 @@ public class TaskNode {
*/ */
private String workerGroup; private String workerGroup;
/**
* worker group id
*/
private Integer workerGroupId;
/** /**
* task time out * task time out
*/ */
@ -325,14 +319,6 @@ public class TaskNode {
this.conditionResult = conditionResult; this.conditionResult = conditionResult;
} }
public Integer getWorkerGroupId() {
return workerGroupId;
}
public void setWorkerGroupId(Integer workerGroupId) {
this.workerGroupId = workerGroupId;
}
public int getDelayTime() { public int getDelayTime() {
return delayTime; return delayTime;
} }
@ -405,7 +391,6 @@ public class TaskNode {
+ ", conditionResult='" + conditionResult + '\'' + ", conditionResult='" + conditionResult + '\''
+ ", taskInstancePriority=" + taskInstancePriority + ", taskInstancePriority=" + taskInstancePriority
+ ", workerGroup='" + workerGroup + '\'' + ", workerGroup='" + workerGroup + '\''
+ ", workerGroupId=" + workerGroupId
+ ", timeout='" + timeout + '\'' + ", timeout='" + timeout + '\''
+ ", delayTime=" + delayTime + ", delayTime=" + delayTime
+ '}'; + '}';

30
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -23,6 +23,8 @@ public class StringUtils {
public static final String EMPTY = ""; public static final String EMPTY = "";
public static final int INDEX_NOT_FOUND = -1;
private StringUtils() { private StringUtils() {
throw new UnsupportedOperationException("Construct StringUtils"); throw new UnsupportedOperationException("Construct StringUtils");
} }
@ -89,4 +91,32 @@ public class StringUtils {
public static boolean equalsIgnoreCase(String str1, String str2) { public static boolean equalsIgnoreCase(String str1, String str2) {
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2); return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
} }
public static String substringBefore(final String str, final String separator) {
if (isEmpty(str) || separator == null) {
return str;
}
if (separator.isEmpty()) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return str;
}
return str.substring(0, pos);
}
public static String substringAfter(final String str, final String separator) {
if (isEmpty(str)) {
return str;
}
if (separator == null) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return EMPTY;
}
return str.substring(pos + separator.length());
}
} }

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProjectUser.java

@ -34,12 +34,9 @@ public class ProjectUser {
@TableField("user_id") @TableField("user_id")
private int userId; private int userId;
@TableField(exist = false) @TableField("project_id")
private int projectId; private int projectId;
@TableField("project_code")
private long projectCode;
/** /**
* project name * project name
*/ */
@ -63,14 +60,6 @@ public class ProjectUser {
@TableField("update_time") @TableField("update_time")
private Date updateTime; private Date updateTime;
public long getProjectCode() {
return projectCode;
}
public void setProjectCode(long projectCode) {
this.projectCode = projectCode;
}
public int getId() { public int getId() {
return id; return id;
} }
@ -141,7 +130,6 @@ public class ProjectUser {
+ "id=" + id + "id=" + id
+ ", userId=" + userId + ", userId=" + userId
+ ", projectId=" + projectId + ", projectId=" + projectId
+ ", projectCode=" + projectCode
+ ", projectName='" + projectName + '\'' + ", projectName='" + projectName + '\''
+ ", userName='" + userName + '\'' + ", userName='" + userName + '\''
+ ", perm=" + perm + ", perm=" + perm

5
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java

@ -55,6 +55,11 @@ public interface ProcessDefinitionLogMapper extends BaseMapper<ProcessDefinition
*/ */
Integer queryMaxVersionForDefinition(@Param("processDefinitionCode") long processDefinitionCode); Integer queryMaxVersionForDefinition(@Param("processDefinitionCode") long processDefinitionCode);
/**
* query max version definition log
*/
ProcessDefinitionLog queryMaxVersionDefinitionLog(@Param("processDefinitionCode") long processDefinitionCode);
/** /**
* query the certain process definition version info by process definition code and version number * query the certain process definition version info by process definition code and version number
* *

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java

@ -85,6 +85,13 @@ public interface ProjectMapper extends BaseMapper<Project> {
*/ */
List<Project> queryAuthedProjectListByUserId(@Param("userId") int userId); List<Project> queryAuthedProjectListByUserId(@Param("userId") int userId);
/**
* query relation project list by userId
* @param userId userId
* @return project list
*/
List<Project> queryRelationProjectListByUserId(@Param("userId") int userId);
/** /**
* query project except userId * query project except userId
* @param userId userId * @param userId userId

14
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.java

@ -17,9 +17,11 @@
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Param;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/** /**
* project user mapper interface * project user mapper interface
*/ */
@ -27,19 +29,21 @@ public interface ProjectUserMapper extends BaseMapper<ProjectUser> {
/** /**
* delte prject user relation * delte prject user relation
* @param projectCode projectCode *
* @param projectId projectId
* @param userId userId * @param userId userId
* @return delete result * @return delete result
*/ */
int deleteProjectRelation(@Param("projectCode") Long projectCode, int deleteProjectRelation(@Param("projectId") int projectId,
@Param("userId") int userId); @Param("userId") int userId);
/** /**
* query project relation * query project relation
* @param projectCode projectCode *
* @param projectId projectId
* @param userId userId * @param userId userId
* @return project user relation * @return project user relation
*/ */
ProjectUser queryProjectRelation(@Param("projectCode") Long projectCode, ProjectUser queryProjectRelation(@Param("projectId") int projectId,
@Param("userId") int userId); @Param("userId") int userId);
} }

66
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java

@ -24,15 +24,21 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.ProcessDag;
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.*;
/** /**
* dag tools * dag tools
*/ */
@ -68,10 +74,10 @@ public class DagHelper {
/** /**
* generate task nodes needed by dag * generate task nodes needed by dag
* *
* @param taskNodeList taskNodeList * @param taskNodeList taskNodeList
* @param startNodeNameList startNodeNameList * @param startNodeNameList startNodeNameList
* @param recoveryNodeNameList recoveryNodeNameList * @param recoveryNodeNameList recoveryNodeNameList
* @param taskDependType taskDependType * @param taskDependType taskDependType
* @return task node list * @return task node list
*/ */
public static List<TaskNode> generateFlowNodeListByStartNode(List<TaskNode> taskNodeList, List<String> startNodeNameList, public static List<TaskNode> generateFlowNodeListByStartNode(List<TaskNode> taskNodeList, List<String> startNodeNameList,
@ -129,7 +135,7 @@ public class DagHelper {
/** /**
* find all the nodes that depended on the start node * find all the nodes that depended on the start node
* *
* @param startNode startNode * @param startNode startNode
* @param taskNodeList taskNodeList * @param taskNodeList taskNodeList
* @return task node list * @return task node list
*/ */
@ -154,9 +160,9 @@ public class DagHelper {
/** /**
* find all nodes that start nodes depend on. * find all nodes that start nodes depend on.
* *
* @param startNode startNode * @param startNode startNode
* @param recoveryNodeNameList recoveryNodeNameList * @param recoveryNodeNameList recoveryNodeNameList
* @param taskNodeList taskNodeList * @param taskNodeList taskNodeList
* @return task node list * @return task node list
*/ */
private static List<TaskNode> getFlowNodeListPre(TaskNode startNode, List<String> recoveryNodeNameList, List<TaskNode> taskNodeList, List<String> visitedNodeNameList) { private static List<TaskNode> getFlowNodeListPre(TaskNode startNode, List<String> recoveryNodeNameList, List<TaskNode> taskNodeList, List<String> visitedNodeNameList) {
@ -189,10 +195,10 @@ public class DagHelper {
/** /**
* generate dag by start nodes and recovery nodes * generate dag by start nodes and recovery nodes
* *
* @param totalTaskNodeList totalTaskNodeList * @param totalTaskNodeList totalTaskNodeList
* @param startNodeNameList startNodeNameList * @param startNodeNameList startNodeNameList
* @param recoveryNodeNameList recoveryNodeNameList * @param recoveryNodeNameList recoveryNodeNameList
* @param depNodeType depNodeType * @param depNodeType depNodeType
* @return process dag * @return process dag
* @throws Exception if error throws Exception * @throws Exception if error throws Exception
*/ */
@ -216,7 +222,7 @@ public class DagHelper {
* find node by node name * find node by node name
* *
* @param nodeDetails nodeDetails * @param nodeDetails nodeDetails
* @param nodeName nodeName * @param nodeName nodeName
* @return task node * @return task node
*/ */
public static TaskNode findNodeByName(List<TaskNode> nodeDetails, String nodeName) { public static TaskNode findNodeByName(List<TaskNode> nodeDetails, String nodeName) {
@ -231,8 +237,8 @@ public class DagHelper {
/** /**
* the task can be submit when all the depends nodes are forbidden or complete * the task can be submit when all the depends nodes are forbidden or complete
* *
* @param taskNode taskNode * @param taskNode taskNode
* @param dag dag * @param dag dag
* @param completeTaskList completeTaskList * @param completeTaskList completeTaskList
* @return can submit * @return can submit
*/ */
@ -262,7 +268,6 @@ public class DagHelper {
* this function parse the condition node to find the right branch. * this function parse the condition node to find the right branch.
* also check all the depends nodes forbidden or complete * also check all the depends nodes forbidden or complete
* *
* @param preNodeName
* @return successor nodes * @return successor nodes
*/ */
public static Set<String> parsePostNodes(String preNodeName, public static Set<String> parsePostNodes(String preNodeName,
@ -299,9 +304,6 @@ public class DagHelper {
/** /**
* if all of the task dependence are skipped, skip it too. * if all of the task dependence are skipped, skip it too.
*
* @param taskNode
* @return
*/ */
private static boolean isTaskNodeNeedSkip(TaskNode taskNode, private static boolean isTaskNodeNeedSkip(TaskNode taskNode,
Map<String, TaskNode> skipTaskNodeList Map<String, TaskNode> skipTaskNodeList
@ -321,9 +323,6 @@ public class DagHelper {
/** /**
* parse condition task find the branch process * parse condition task find the branch process
* set skip flag for another one. * set skip flag for another one.
*
* @param nodeName
* @return
*/ */
public static List<String> parseConditionTask(String nodeName, public static List<String> parseConditionTask(String nodeName,
Map<String, TaskNode> skipTaskNodeList, Map<String, TaskNode> skipTaskNodeList,
@ -358,11 +357,6 @@ public class DagHelper {
/** /**
* set task node and the post nodes skip flag * set task node and the post nodes skip flag
*
* @param skipNodeName
* @param dag
* @param completeTaskList
* @param skipTaskNodeList
*/ */
private static void setTaskNodeSkip(String skipNodeName, private static void setTaskNodeSkip(String skipNodeName,
DAG<String, TaskNode, TaskNodeRelation> dag, DAG<String, TaskNode, TaskNodeRelation> dag,
@ -451,9 +445,13 @@ public class DagHelper {
List<TaskNodeRelation> taskNodeRelations = new ArrayList<>(); List<TaskNodeRelation> taskNodeRelations = new ArrayList<>();
for (ProcessTaskRelation processTaskRelation : processTaskRelations) { for (ProcessTaskRelation processTaskRelation : processTaskRelations) {
if (processTaskRelation.getPreTaskCode() != 0) { long preTaskCode = processTaskRelation.getPreTaskCode();
TaskNode preNode = taskNodeMap.get(processTaskRelation.getPreTaskCode()); long postTaskCode = processTaskRelation.getPostTaskCode();
TaskNode postNode = taskNodeMap.get(processTaskRelation.getPostTaskCode());
if (processTaskRelation.getPreTaskCode() != 0
&& taskNodeMap.containsKey(preTaskCode) && taskNodeMap.containsKey(postTaskCode)) {
TaskNode preNode = taskNodeMap.get(preTaskCode);
TaskNode postNode = taskNodeMap.get(postTaskCode);
taskNodeRelations.add(new TaskNodeRelation(preNode.getName(), postNode.getName())); taskNodeRelations.add(new TaskNodeRelation(preNode.getName(), postNode.getName()));
} }
} }
@ -465,9 +463,6 @@ public class DagHelper {
/** /**
* is there have conditions after the parent node * is there have conditions after the parent node
*
* @param parentNodeName
* @return
*/ */
public static boolean haveConditionsAfterNode(String parentNodeName, public static boolean haveConditionsAfterNode(String parentNodeName,
DAG<String, TaskNode, TaskNodeRelation> dag DAG<String, TaskNode, TaskNodeRelation> dag
@ -489,9 +484,6 @@ public class DagHelper {
/** /**
* is there have conditions after the parent node * is there have conditions after the parent node
*
* @param parentNodeName
* @return
*/ */
public static boolean haveConditionsAfterNode(String parentNodeName, public static boolean haveConditionsAfterNode(String parentNodeName,
List<TaskNode> taskNodes List<TaskNode> taskNodes

2
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.xml

@ -29,7 +29,7 @@
</foreach> </foreach>
</if> </if>
<if test="startTime != null and endTime != null"> <if test="startTime != null and endTime != null">
and cmd.startTime <![CDATA[ >= ]]> #{startTime} and cmd.update_time <![CDATA[ <= ]]> #{endTime} and cmd.start_time <![CDATA[ >= ]]> #{startTime} and cmd.update_time <![CDATA[ <= ]]> #{endTime}
</if> </if>
group by cmd.command_type group by cmd.command_type
</select> </select>

8
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.xml

@ -56,6 +56,14 @@
where code = #{processDefinitionCode} where code = #{processDefinitionCode}
</select> </select>
<select id="queryMaxVersionDefinitionLog" resultType="org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog">
select
<include refid="baseSql"/>
from t_ds_process_definition_log
where code = #{processDefinitionCode} order by version desc limit 1
</select>
<select id="queryProcessDefinitionVersionsPaging" <select id="queryProcessDefinitionVersionsPaging"
resultType="org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog"> resultType="org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog">
select select

29
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml

@ -19,7 +19,6 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectMapper"> <mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectMapper">
<sql id="baseSql"> <sql id="baseSql">
id, name, code, description, user_id, flag, create_time, update_time id, name, code, description, user_id, flag, create_time, update_time
</sql> </sql>
<sql id="baseSqlV2"> <sql id="baseSqlV2">
@ -83,9 +82,9 @@
join t_ds_user u on u.id=p.user_id join t_ds_user u on u.id=p.user_id
where 1=1 where 1=1
<if test="userId != 0"> <if test="userId != 0">
and p.code in and p.id in
(select project_code from t_ds_relation_project_user where user_id=#{userId} (select project_id from t_ds_relation_project_user where user_id=#{userId}
union select id as project_code from t_ds_project where user_id=#{userId} union select id as project_id from t_ds_project where user_id=#{userId}
) )
</if> </if>
<if test="searchName!=null and searchName != ''"> <if test="searchName!=null and searchName != ''">
@ -101,6 +100,18 @@
from t_ds_project p,t_ds_relation_project_user rel from t_ds_project p,t_ds_relation_project_user rel
where p.id = rel.project_id and rel.user_id= #{userId} where p.id = rel.project_id and rel.user_id= #{userId}
</select> </select>
<select id="queryRelationProjectListByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select
<include refid="baseSqlV2">
<property name="alias" value="p"/>
</include>
from t_ds_project p left join t_ds_relation_project_user rel
on p.id = rel.project_id
where 1=1
<if test="userId != 0 ">
and rel.user_id= #{userId}
</if>
</select>
<select id="queryProjectExceptUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project"> <select id="queryProjectExceptUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select select
<include refid="baseSql"/> <include refid="baseSql"/>
@ -114,16 +125,18 @@
where user_id = #{userId} where user_id = #{userId}
</select> </select>
<select id="queryProjectCreatedAndAuthorizedByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project"> <select id="queryProjectCreatedAndAuthorizedByUserId" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select * from t_ds_project where id in select
<include refid="baseSql"/>
from t_ds_project where id in
(select project_id from t_ds_relation_project_user where user_id=#{userId} (select project_id from t_ds_relation_project_user where user_id=#{userId}
union select id as project_id from t_ds_project where user_id=#{userId}) union select id as project_id from t_ds_project where user_id=#{userId})
</select> </select>
<select id="queryProjectWithUserByProcessInstanceId" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser"> <select id="queryProjectWithUserByProcessInstanceId" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser">
select select
dp.code projectCode, dp.id project_id,
dp.name projectName, dp.name project_name,
u.user_name userName u.user_name user_name
from t_ds_process_instance di from t_ds_process_instance di
join t_ds_process_definition dpd on di.process_definition_code = dpd.code join t_ds_process_definition dpd on di.process_definition_code = dpd.code
join t_ds_project dp on dpd.project_code = dp.code join t_ds_project dp on dpd.project_code = dp.code

8
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapper.xml

@ -19,21 +19,21 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper"> <mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper">
<sql id="baseSql"> <sql id="baseSql">
id, user_id, project_code, perm, create_time, update_time id, user_id, project_id, perm, create_time, update_time
</sql> </sql>
<delete id="deleteProjectRelation"> <delete id="deleteProjectRelation">
delete from t_ds_relation_project_user delete from t_ds_relation_project_user
where 1=1 where 1=1
and user_id = #{userId} and user_id = #{userId}
<if test="projectCode != 0 "> <if test="projectId != 0 ">
and project_code = #{projectCode} and project_id = #{projectId}
</if> </if>
</delete> </delete>
<select id="queryProjectRelation" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser"> <select id="queryProjectRelation" resultType="org.apache.dolphinscheduler.dao.entity.ProjectUser">
select select
<include refid="baseSql"/> <include refid="baseSql"/>
from t_ds_relation_project_user from t_ds_relation_project_user
where project_code = #{projectCode} where project_id = #{projectId}
and user_id = #{userId} and user_id = #{userId}
limit 1 limit 1
</select> </select>

23
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapperTest.java

@ -107,7 +107,7 @@ public class ProcessDefinitionLogMapperTest {
@Test @Test
public void testQueryByDefinitionName() { public void testQueryByDefinitionName() {
ProcessDefinitionLog processDefinitionLog = insertOne(); insertOne();
Project project = new Project(); Project project = new Project();
project.setCode(1L); project.setCode(1L);
project.setName("ut project"); project.setName("ut project");
@ -130,7 +130,7 @@ public class ProcessDefinitionLogMapperTest {
@Test @Test
public void testQueryByDefinitionCode() { public void testQueryByDefinitionCode() {
ProcessDefinitionLog processDefinitionLog = insertOne(); insertOne();
List<ProcessDefinitionLog> processDefinitionLogs = processDefinitionLogMapper List<ProcessDefinitionLog> processDefinitionLogs = processDefinitionLogMapper
.queryByDefinitionCode(1L); .queryByDefinitionCode(1L);
@ -139,7 +139,7 @@ public class ProcessDefinitionLogMapperTest {
@Test @Test
public void testQueryByDefinitionCodeAndVersion() { public void testQueryByDefinitionCodeAndVersion() {
ProcessDefinitionLog processDefinitionLog = insertOne(); insertOne();
ProcessDefinitionLog processDefinitionLogs = processDefinitionLogMapper ProcessDefinitionLog processDefinitionLogs = processDefinitionLogMapper
.queryByDefinitionCodeAndVersion(1L, 1); .queryByDefinitionCodeAndVersion(1L, 1);
@ -148,8 +148,8 @@ public class ProcessDefinitionLogMapperTest {
@Test @Test
public void testQueryMaxVersionForDefinition() { public void testQueryMaxVersionForDefinition() {
ProcessDefinitionLog processDefinitionLog = insertOne(); insertOne();
ProcessDefinitionLog processDefinitionLog1 = insertTwo(); insertTwo();
Integer version = processDefinitionLogMapper.queryMaxVersionForDefinition(1L); Integer version = processDefinitionLogMapper.queryMaxVersionForDefinition(1L);
Assert.assertEquals(2, version == null ? 1 : version); Assert.assertEquals(2, version == null ? 1 : version);
@ -157,7 +157,7 @@ public class ProcessDefinitionLogMapperTest {
@Test @Test
public void testQueryProcessDefinitionVersionsPaging() { public void testQueryProcessDefinitionVersionsPaging() {
ProcessDefinitionLog processDefinitionLog = insertOne(); insertOne();
Page<ProcessDefinitionLog> page = new Page(1, 3); Page<ProcessDefinitionLog> page = new Page(1, 3);
IPage<ProcessDefinitionLog> processDefinitionLogs = processDefinitionLogMapper.queryProcessDefinitionVersionsPaging(page, 1L); IPage<ProcessDefinitionLog> processDefinitionLogs = processDefinitionLogMapper.queryProcessDefinitionVersionsPaging(page, 1L);
Assert.assertNotEquals(processDefinitionLogs.getTotal(), 0); Assert.assertNotEquals(processDefinitionLogs.getTotal(), 0);
@ -165,10 +165,19 @@ public class ProcessDefinitionLogMapperTest {
@Test @Test
public void testDeleteByProcessDefinitionCodeAndVersion() { public void testDeleteByProcessDefinitionCodeAndVersion() {
ProcessDefinitionLog processDefinitionLog = insertOne(); insertOne();
Page<ProcessDefinitionLog> page = new Page(1, 3); Page<ProcessDefinitionLog> page = new Page(1, 3);
int processDefinitionLogs = processDefinitionLogMapper.deleteByProcessDefinitionCodeAndVersion(1L, 1); int processDefinitionLogs = processDefinitionLogMapper.deleteByProcessDefinitionCodeAndVersion(1L, 1);
Assert.assertNotEquals(processDefinitionLogs, 0); Assert.assertNotEquals(processDefinitionLogs, 0);
} }
@Test
public void testQueryMaxVersionDefinitionLog() {
insertOne();
insertTwo();
ProcessDefinitionLog processDefinitionLog2 = processDefinitionLogMapper.queryMaxVersionDefinitionLog(1L);
Assert.assertEquals(2, processDefinitionLog2.getVersion());
}
} }

5
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java

@ -52,7 +52,6 @@ public class ProjectUserMapperTest {
private ProjectUser insertOne(){ private ProjectUser insertOne(){
//insertOne //insertOne
ProjectUser projectUser = new ProjectUser(); ProjectUser projectUser = new ProjectUser();
projectUser.setProjectCode(1010L);
projectUser.setProjectId(1010); projectUser.setProjectId(1010);
projectUser.setUserId(111); projectUser.setUserId(111);
projectUserMapper.insert(projectUser); projectUserMapper.insert(projectUser);
@ -101,7 +100,7 @@ public class ProjectUserMapperTest {
ProjectUser projectUser = insertOne(); ProjectUser projectUser = insertOne();
int delete = projectUserMapper.deleteProjectRelation(projectUser.getProjectCode(), projectUser.getUserId()); int delete = projectUserMapper.deleteProjectRelation(projectUser.getProjectId(), projectUser.getUserId());
assertThat(delete,greaterThanOrEqualTo(1)); assertThat(delete,greaterThanOrEqualTo(1));
} }
@ -112,7 +111,7 @@ public class ProjectUserMapperTest {
@Test @Test
public void testQueryProjectRelation() { public void testQueryProjectRelation() {
ProjectUser projectUser = insertOne(); ProjectUser projectUser = insertOne();
ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectCode(), projectUser.getUserId()); ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId());
Assert.assertNotEquals(projectUser1, null); Assert.assertNotEquals(projectUser1, null);
} }

15
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -2222,7 +2222,6 @@ public class ProcessService {
* update task definition * update task definition
*/ */
public int updateTaskDefinition(User operator, Long projectCode, TaskNode taskNode, TaskDefinition taskDefinition) { public int updateTaskDefinition(User operator, Long projectCode, TaskNode taskNode, TaskDefinition taskDefinition) {
Integer version = taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode()); Integer version = taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode());
Date now = new Date(); Date now = new Date();
taskDefinition.setProjectCode(projectCode); taskDefinition.setProjectCode(projectCode);
@ -2240,10 +2239,11 @@ public class ProcessService {
} }
private void setTaskFromTaskNode(TaskNode taskNode, TaskDefinition taskDefinition) { private void setTaskFromTaskNode(TaskNode taskNode, TaskDefinition taskDefinition) {
taskDefinition.setName(taskNode.getName()); // TODO for the front-end UI, name with id
taskDefinition.setName(taskNode.getId() + "|" + taskNode.getName());
taskDefinition.setDescription(taskNode.getDesc()); taskDefinition.setDescription(taskNode.getDesc());
taskDefinition.setTaskType(TaskType.of(taskNode.getType())); taskDefinition.setTaskType(TaskType.of(taskNode.getType()));
taskDefinition.setTaskParams(taskNode.getParams()); taskDefinition.setTaskParams(TaskType.of(taskNode.getType()) == TaskType.DEPENDENT ? taskNode.getDependence() : taskNode.getParams());
taskDefinition.setFlag(taskNode.isForbidden() ? Flag.NO : Flag.YES); taskDefinition.setFlag(taskNode.isForbidden() ? Flag.NO : Flag.YES);
taskDefinition.setTaskPriority(taskNode.getTaskInstancePriority()); taskDefinition.setTaskPriority(taskNode.getTaskInstancePriority());
taskDefinition.setWorkerGroup(taskNode.getWorkerGroup()); taskDefinition.setWorkerGroup(taskNode.getWorkerGroup());
@ -2298,6 +2298,7 @@ public class ProcessService {
String desc, String locations, String connects) { String desc, String locations, String connects) {
ProcessDefinitionLog processDefinitionLog = new ProcessDefinitionLog(); ProcessDefinitionLog processDefinitionLog = new ProcessDefinitionLog();
Integer version = processDefineLogMapper.queryMaxVersionForDefinition(processDefinitionCode); Integer version = processDefineLogMapper.queryMaxVersionForDefinition(processDefinitionCode);
processDefinitionLog.setUserId(operator.getId());
processDefinitionLog.setCode(processDefinitionCode); processDefinitionLog.setCode(processDefinitionCode);
processDefinitionLog.setVersion(version == null || version == 0 ? 1 : version); processDefinitionLog.setVersion(version == null || version == 0 ? 1 : version);
processDefinitionLog.setName(processDefinitionName); processDefinitionLog.setName(processDefinitionName);
@ -2500,15 +2501,17 @@ public class ProcessService {
Map<Long, TaskDefinitionLog> taskDefinitionLogMap = taskDefinitionLogs.stream().collect(Collectors.toMap(TaskDefinitionLog::getCode, log -> log)); Map<Long, TaskDefinitionLog> taskDefinitionLogMap = taskDefinitionLogs.stream().collect(Collectors.toMap(TaskDefinitionLog::getCode, log -> log));
taskNodeMap.forEach((k, v) -> { taskNodeMap.forEach((k, v) -> {
TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMap.get(k); TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMap.get(k);
v.setId("task-" + taskDefinitionLog.getId()); // TODO split from name
v.setId(StringUtils.substringBefore(taskDefinitionLog.getName(), "|"));
v.setCode(taskDefinitionLog.getCode()); v.setCode(taskDefinitionLog.getCode());
v.setName(taskDefinitionLog.getName()); v.setName(StringUtils.substringAfter(taskDefinitionLog.getName(), "|"));
v.setDesc(taskDefinitionLog.getDescription()); v.setDesc(taskDefinitionLog.getDescription());
v.setType(taskDefinitionLog.getTaskType().getDescp().toUpperCase()); v.setType(taskDefinitionLog.getTaskType().getDescp().toUpperCase());
v.setRunFlag(taskDefinitionLog.getFlag() == Flag.YES ? Constants.FLOWNODE_RUN_FLAG_NORMAL : Constants.FLOWNODE_RUN_FLAG_FORBIDDEN); v.setRunFlag(taskDefinitionLog.getFlag() == Flag.YES ? Constants.FLOWNODE_RUN_FLAG_NORMAL : Constants.FLOWNODE_RUN_FLAG_FORBIDDEN);
v.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes()); v.setMaxRetryTimes(taskDefinitionLog.getFailRetryTimes());
v.setRetryInterval(taskDefinitionLog.getFailRetryInterval()); v.setRetryInterval(taskDefinitionLog.getFailRetryInterval());
v.setParams(taskDefinitionLog.getTaskParams()); v.setParams(taskDefinitionLog.getTaskType() == TaskType.DEPENDENT ? "" : taskDefinitionLog.getTaskParams());
v.setDependence(taskDefinitionLog.getTaskType() == TaskType.DEPENDENT ? taskDefinitionLog.getTaskParams() : null);
v.setTaskInstancePriority(taskDefinitionLog.getTaskPriority()); v.setTaskInstancePriority(taskDefinitionLog.getTaskPriority());
v.setWorkerGroup(taskDefinitionLog.getWorkerGroup()); v.setWorkerGroup(taskDefinitionLog.getWorkerGroup());
v.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN, v.setTimeout(JSONUtils.toJsonString(new TaskTimeoutParameter(taskDefinitionLog.getTimeoutFlag() == TimeoutFlag.OPEN,

179
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

@ -21,27 +21,37 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PRO
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_ID; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_ID;
import static org.mockito.ArgumentMatchers.any;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.ProcessData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtilsTest; import org.apache.dolphinscheduler.service.quartz.cron.CronUtilsTest;
@ -87,7 +97,13 @@ public class ProcessServiceTest {
@Mock @Mock
private UserMapper userMapper; private UserMapper userMapper;
@Mock @Mock
TaskInstanceMapper taskInstanceMapper; private TaskInstanceMapper taskInstanceMapper;
@Mock
private TaskDefinitionLogMapper taskDefinitionLogMapper;
@Mock
private ProcessTaskRelationMapper processTaskRelationMapper;
@Mock
private ProcessDefinitionLogMapper processDefineLogMapper;
@Test @Test
public void testCreateSubCommand() { public void testCreateSubCommand() {
@ -328,4 +344,163 @@ public class ProcessServiceTest {
processService.recurseFindSubProcessId(parentId, ids); processService.recurseFindSubProcessId(parentId, ids);
} }
@Test
public void testSaveProcessDefinition() {
User user = new User();
user.setId(1);
Project project = new Project();
project.setCode(1L);
ProcessData processData = new ProcessData();
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setCode(1L);
processDefinition.setId(123);
processDefinition.setName("test");
processDefinition.setVersion(1);
processDefinition.setCode(11L);
Mockito.when(processDefineMapper.updateById(any())).thenReturn(1);
Mockito.when(processDefineLogMapper.insert(any())).thenReturn(1);
int i = processService.saveProcessDefinition(user, project, "name", "desc", "locations", "connects", processData, processDefinition);
Assert.assertEquals(1, i);
}
@Test
public void testSwitchVersion() {
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setCode(1L);
processDefinition.setId(123);
processDefinition.setName("test");
processDefinition.setVersion(1);
processDefinition.setCode(11L);
ProcessDefinitionLog processDefinitionLog = new ProcessDefinitionLog();
processDefinitionLog.setCode(1L);
Mockito.when(processDefineMapper.updateById(any())).thenReturn(1);
int i = processService.switchVersion(processDefinition, processDefinitionLog);
Assert.assertEquals(1, i);
}
@Test
public void testGenDagGraph() {
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setCode(1L);
processDefinition.setId(123);
processDefinition.setName("test");
processDefinition.setVersion(1);
processDefinition.setCode(11L);
ProcessTaskRelationLog processTaskRelationLog = new ProcessTaskRelationLog();
processTaskRelationLog.setName("def 1");
processTaskRelationLog.setProcessDefinitionVersion(1);
processTaskRelationLog.setProjectCode(1L);
processTaskRelationLog.setProcessDefinitionCode(1L);
processTaskRelationLog.setPostTaskCode(3L);
processTaskRelationLog.setPreTaskCode(2L);
processTaskRelationLog.setUpdateTime(new Date());
processTaskRelationLog.setCreateTime(new Date());
List<ProcessTaskRelationLog> list = new ArrayList<>();
list.add(processTaskRelationLog);
TaskDefinitionLog taskDefinition = new TaskDefinitionLog();
taskDefinition.setCode(3L);
taskDefinition.setName("1-test");
taskDefinition.setProjectCode(1L);
taskDefinition.setTaskType(TaskType.SHELL);
taskDefinition.setUserId(1);
taskDefinition.setVersion(2);
taskDefinition.setCreateTime(new Date());
taskDefinition.setUpdateTime(new Date());
TaskDefinitionLog td2 = new TaskDefinitionLog();
td2.setCode(2L);
td2.setName("unit-test");
td2.setProjectCode(1L);
td2.setTaskType(TaskType.SHELL);
td2.setUserId(1);
td2.setVersion(1);
td2.setCreateTime(new Date());
td2.setUpdateTime(new Date());
List<TaskDefinitionLog> taskDefinitionLogs = new ArrayList<>();
taskDefinitionLogs.add(taskDefinition);
taskDefinitionLogs.add(td2);
Mockito.when(taskDefinitionLogMapper.queryByTaskDefinitions(any())).thenReturn(taskDefinitionLogs);
Mockito.when(processTaskRelationLogMapper.queryByProcessCodeAndVersion(Mockito.anyLong(), Mockito.anyInt())).thenReturn(list);
DAG<String, TaskNode, TaskNodeRelation> stringTaskNodeTaskNodeRelationDAG = processService.genDagGraph(processDefinition);
Assert.assertNotEquals(0, stringTaskNodeTaskNodeRelationDAG.getNodesCount());
}
@Test
public void testGenProcessData() {
String processDefinitionJson = "{\"tasks\":[{\"id\":\"task-0\",\"code\":3,\"version\":0,\"name\":\"1-test\""
+ ",\"desc\":null,\"type\":\"SHELL\",\"runFlag\":\"FORBIDDEN\",\"loc\":null,\"maxRetryTimes\":0"
+ ",\"retryInterval\":0,\"params\":null,\"preTasks\":[\"unit-test\"],\"preTaskNodeList\":[{\"code\":2"
+ ",\"name\":\"unit-test\",\"version\":0}],\"extras\":null,\"depList\":[\"unit-test\"],\"dependence\":null"
+ ",\"conditionResult\":null,\"taskInstancePriority\":null,\"workerGroup\":null,\"workerGroupId\":null"
+ ",\"timeout\":{\"enable\":false,\"strategy\":null,\"interval\":0},\"delayTime\":0}]"
+ ",\"globalParams\":[],\"timeout\":0,\"tenantId\":0}";
ProcessDefinition processDefinition = new ProcessDefinition();
processDefinition.setCode(1L);
processDefinition.setId(123);
processDefinition.setName("test");
processDefinition.setVersion(1);
processDefinition.setCode(11L);
ProcessTaskRelationLog processTaskRelationLog = new ProcessTaskRelationLog();
processTaskRelationLog.setName("def 1");
processTaskRelationLog.setProcessDefinitionVersion(1);
processTaskRelationLog.setProjectCode(1L);
processTaskRelationLog.setProcessDefinitionCode(1L);
processTaskRelationLog.setPostTaskCode(3L);
processTaskRelationLog.setPreTaskCode(2L);
processTaskRelationLog.setUpdateTime(new Date());
processTaskRelationLog.setCreateTime(new Date());
List<ProcessTaskRelationLog> list = new ArrayList<>();
list.add(processTaskRelationLog);
TaskDefinitionLog taskDefinition = new TaskDefinitionLog();
taskDefinition.setCode(3L);
taskDefinition.setName("1-test");
taskDefinition.setProjectCode(1L);
taskDefinition.setTaskType(TaskType.SHELL);
taskDefinition.setUserId(1);
taskDefinition.setVersion(2);
taskDefinition.setCreateTime(new Date());
taskDefinition.setUpdateTime(new Date());
TaskDefinitionLog td2 = new TaskDefinitionLog();
td2.setCode(2L);
td2.setName("unit-test");
td2.setProjectCode(1L);
td2.setTaskType(TaskType.SHELL);
td2.setUserId(1);
td2.setVersion(1);
td2.setCreateTime(new Date());
td2.setUpdateTime(new Date());
List<TaskDefinitionLog> taskDefinitionLogs = new ArrayList<>();
taskDefinitionLogs.add(taskDefinition);
taskDefinitionLogs.add(td2);
Mockito.when(taskDefinitionLogMapper.queryByTaskDefinitions(any())).thenReturn(taskDefinitionLogs);
Mockito.when(processTaskRelationLogMapper.queryByProcessCodeAndVersion(Mockito.anyLong(), Mockito.anyInt())).thenReturn(list);
String json = JSONUtils.toJsonString(processService.genProcessData(processDefinition));
Assert.assertEquals(processDefinitionJson, json);
}
} }

18
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue

@ -64,7 +64,7 @@
:rows="2" :rows="2"
type="textarea" type="textarea"
:disabled="isDetails" :disabled="isDetails"
v-model="description" v-model="desc"
:placeholder="$t('Please enter description')"> :placeholder="$t('Please enter description')">
</el-input> </el-input>
</div> </div>
@ -314,7 +314,7 @@
// node name // node name
name: '', name: '',
// description // description
description: '', desc: '',
// Node echo data // Node echo data
backfillItem: {}, backfillItem: {},
cacheBackfillItem: {}, cacheBackfillItem: {},
@ -332,8 +332,8 @@
dependence: {}, dependence: {},
// cache dependence // cache dependence
cacheDependence: {}, cacheDependence: {},
//task code // task code
code:'', code: '',
// Current node params data // Current node params data
params: {}, params: {},
// Running sign // Running sign
@ -475,9 +475,9 @@
type: this.nodeData.taskType, type: this.nodeData.taskType,
id: this.nodeData.id, id: this.nodeData.id,
name: this.name, name: this.name,
code:this.code, code: this.code,
params: this.params, params: this.params,
description: this.description, desc: this.desc,
runFlag: this.runFlag, runFlag: this.runFlag,
conditionResult: this.conditionResult, conditionResult: this.conditionResult,
dependence: this.cacheDependence, dependence: this.cacheDependence,
@ -601,7 +601,7 @@
name: this.name, name: this.name,
code: this.code, code: this.code,
params: this.params, params: this.params,
description: this.description, desc: this.desc,
runFlag: this.runFlag, runFlag: this.runFlag,
conditionResult: this.conditionResult, conditionResult: this.conditionResult,
dependence: this.dependence, dependence: this.dependence,
@ -696,7 +696,7 @@
this.name = o.name this.name = o.name
this.taskInstancePriority = o.taskInstancePriority this.taskInstancePriority = o.taskInstancePriority
this.runFlag = o.runFlag || 'NORMAL' this.runFlag = o.runFlag || 'NORMAL'
this.description = o.description this.desc = o.desc
this.maxRetryTimes = o.maxRetryTimes this.maxRetryTimes = o.maxRetryTimes
this.retryInterval = o.retryInterval this.retryInterval = o.retryInterval
this.delayTime = o.delayTime this.delayTime = o.delayTime
@ -767,7 +767,7 @@
type: this.nodeData.taskType, type: this.nodeData.taskType,
id: this.nodeData.id, id: this.nodeData.id,
name: this.name, name: this.name,
description: this.description, desc: this.desc,
runFlag: this.runFlag, runFlag: this.runFlag,
dependence: this.cacheDependence, dependence: this.cacheDependence,
maxRetryTimes: this.maxRetryTimes, maxRetryTimes: this.maxRetryTimes,

2
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/index.vue

@ -29,7 +29,7 @@
</a> </a>
</div> </div>
</div> </div>
<template v-show="!isNodata"> <template v-if="!isNodata">
<div class="gantt"></div> <div class="gantt"></div>
</template> </template>
<template v-if="isNodata"> <template v-if="isNodata">

2
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/edit/index.vue

@ -21,7 +21,7 @@
<h2> <h2>
<span>{{name}}</span> <span>{{name}}</span>
</h2> </h2>
<template v-show="isViewType"> <template v-if="isViewType">
<template v-if="!msg"> <template v-if="!msg">
<div class="code-mirror-model"> <div class="code-mirror-model">
<textarea id="code-edit-mirror" name="code-edit-mirror"></textarea> <textarea id="code-edit-mirror" name="code-edit-mirror"></textarea>

2
sql/dolphinscheduler_mysql.sql

@ -696,7 +696,7 @@ DROP TABLE IF EXISTS `t_ds_relation_project_user`;
CREATE TABLE `t_ds_relation_project_user` ( CREATE TABLE `t_ds_relation_project_user` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
`user_id` int(11) NOT NULL COMMENT 'user id', `user_id` int(11) NOT NULL COMMENT 'user id',
`project_code` bigint(20) DEFAULT NULL COMMENT 'project code', `project_id` int(11) DEFAULT NULL COMMENT 'project id',
`perm` int(11) DEFAULT '1' COMMENT 'limits of authority', `perm` int(11) DEFAULT '1' COMMENT 'limits of authority',
`create_time` datetime DEFAULT NULL COMMENT 'create time', `create_time` datetime DEFAULT NULL COMMENT 'create time',
`update_time` datetime DEFAULT NULL COMMENT 'update time', `update_time` datetime DEFAULT NULL COMMENT 'update time',

Loading…
Cancel
Save