xiangzihao
4 months ago
committed by
GitHub
52 changed files with 1524 additions and 1151 deletions
@ -0,0 +1,29 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.dto; |
||||
|
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
|
||||
@Data |
||||
@NoArgsConstructor |
||||
public class TaskCodeVersionDto { |
||||
|
||||
private long code; |
||||
private int version; |
||||
} |
||||
@ -0,0 +1,307 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.service.impl; |
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status; |
||||
import org.apache.dolphinscheduler.api.exceptions.ServiceException; |
||||
import org.apache.dolphinscheduler.api.service.ProcessLineageService; |
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.dao.entity.DependentLineageTask; |
||||
import org.apache.dolphinscheduler.dao.entity.DependentProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.Project; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.repository.ProcessTaskLineageDao; |
||||
|
||||
import java.text.MessageFormat; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Optional; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
import org.springframework.util.CollectionUtils; |
||||
|
||||
/** |
||||
* work flow lineage service impl |
||||
*/ |
||||
@Slf4j |
||||
@Service |
||||
public class ProcessLineageServiceImpl extends BaseServiceImpl implements ProcessLineageService { |
||||
|
||||
@Autowired |
||||
private ProjectMapper projectMapper; |
||||
|
||||
@Autowired |
||||
private TaskDefinitionLogMapper taskDefinitionLogMapper; |
||||
|
||||
@Autowired |
||||
private TaskDefinitionMapper taskDefinitionMapper; |
||||
|
||||
@Autowired |
||||
private ProcessTaskLineageDao processTaskLineageDao; |
||||
@Autowired |
||||
private ProcessDefinitionMapper processDefinitionMapper; |
||||
|
||||
@Override |
||||
public List<WorkFlowRelationDetail> queryWorkFlowLineageByName(long projectCode, String processDefinitionName) { |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
throw new ServiceException(Status.PROJECT_NOT_FOUND, projectCode); |
||||
} |
||||
return processTaskLineageDao.queryWorkFlowLineageByName(projectCode, processDefinitionName); |
||||
} |
||||
|
||||
@Override |
||||
public WorkFlowLineage queryWorkFlowLineageByCode(long projectCode, long processDefinitionCode) { |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
throw new ServiceException(Status.PROJECT_NOT_FOUND, projectCode); |
||||
} |
||||
List<ProcessTaskLineage> upstreamProcessTaskLineageList = |
||||
processTaskLineageDao.queryByProcessDefinitionCode(processDefinitionCode); |
||||
List<ProcessTaskLineage> downstreamProcessTaskLineageList = |
||||
processTaskLineageDao.queryWorkFlowLineageByDept(projectCode, processDefinitionCode, |
||||
Constants.DEPENDENT_ALL_TASK); |
||||
List<ProcessTaskLineage> totalProcessTaskLineageList = |
||||
Stream.of(upstreamProcessTaskLineageList, downstreamProcessTaskLineageList) |
||||
.flatMap(List::stream) |
||||
.collect(Collectors.toList()); |
||||
|
||||
List<WorkFlowRelation> workFlowRelationList = getWorkFlowRelations(totalProcessTaskLineageList); |
||||
List<WorkFlowRelationDetail> workFlowRelationDetailList = |
||||
getWorkflowRelationDetails(totalProcessTaskLineageList.stream() |
||||
.flatMap(pl -> { |
||||
List<Long> processDefinitionCodes = new ArrayList<>(); |
||||
processDefinitionCodes.add(pl.getProcessDefinitionCode()); |
||||
processDefinitionCodes.add(pl.getDeptProcessDefinitionCode()); |
||||
return processDefinitionCodes.stream(); |
||||
}).distinct().collect(Collectors.toList())); |
||||
|
||||
WorkFlowLineage workFlowLineage = new WorkFlowLineage(); |
||||
workFlowLineage.setWorkFlowRelationDetailList(workFlowRelationDetailList); |
||||
workFlowLineage.setWorkFlowRelationList(workFlowRelationList); |
||||
return workFlowLineage; |
||||
} |
||||
|
||||
@Override |
||||
public WorkFlowLineage queryWorkFlowLineage(long projectCode) { |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
throw new ServiceException(Status.PROJECT_NOT_FOUND, projectCode); |
||||
} |
||||
List<ProcessTaskLineage> processTaskLineageList = processTaskLineageDao.queryByProjectCode(projectCode); |
||||
List<WorkFlowRelation> workFlowRelationList = getWorkFlowRelations(processTaskLineageList); |
||||
List<WorkFlowRelationDetail> workFlowRelationDetailList = |
||||
getWorkflowRelationDetails(processTaskLineageList.stream() |
||||
.flatMap(pl -> { |
||||
List<Long> processDefinitionCodes = new ArrayList<>(); |
||||
processDefinitionCodes.add(pl.getProcessDefinitionCode()); |
||||
processDefinitionCodes.add(pl.getDeptProcessDefinitionCode()); |
||||
return processDefinitionCodes.stream(); |
||||
}).distinct().collect(Collectors.toList())); |
||||
|
||||
WorkFlowLineage workFlowLineage = new WorkFlowLineage(); |
||||
workFlowLineage.setWorkFlowRelationList(workFlowRelationList); |
||||
workFlowLineage.setWorkFlowRelationDetailList(workFlowRelationDetailList); |
||||
return workFlowLineage; |
||||
} |
||||
|
||||
private List<WorkFlowRelation> getWorkFlowRelations(List<ProcessTaskLineage> processTaskLineageList) { |
||||
List<WorkFlowRelation> workFlowRelations = new ArrayList<>(); |
||||
List<Long> processDefinitionCodes = processTaskLineageList.stream() |
||||
.map(ProcessTaskLineage::getProcessDefinitionCode).distinct().collect(Collectors.toList()); |
||||
for (ProcessTaskLineage processTaskLineage : processTaskLineageList) { |
||||
workFlowRelations.add(new WorkFlowRelation(processTaskLineage.getDeptProcessDefinitionCode(), |
||||
processTaskLineage.getProcessDefinitionCode())); |
||||
|
||||
if (!processDefinitionCodes.contains(processTaskLineage.getDeptProcessDefinitionCode())) { |
||||
workFlowRelations.add(new WorkFlowRelation(0, processTaskLineage.getProcessDefinitionCode())); |
||||
} |
||||
} |
||||
return workFlowRelations; |
||||
} |
||||
|
||||
private List<WorkFlowRelationDetail> getWorkflowRelationDetails(List<Long> processDefinitionCodes) { |
||||
List<WorkFlowRelationDetail> workFlowRelationDetails = new ArrayList<>(); |
||||
for (Long processDefinitionCode : processDefinitionCodes) { |
||||
List<WorkFlowRelationDetail> workFlowRelationDetailList = |
||||
processTaskLineageDao.queryWorkFlowLineageByCode(processDefinitionCode); |
||||
workFlowRelationDetails.addAll(workFlowRelationDetailList); |
||||
} |
||||
return workFlowRelationDetails; |
||||
} |
||||
|
||||
/** |
||||
* Query tasks depend on process definition, include upstream or downstream |
||||
* and return tasks dependence with string format. |
||||
* |
||||
* @param projectCode Project code want to query tasks dependence |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @param taskCode Task code want to query tasks dependence |
||||
* @return Optional of formatter message |
||||
*/ |
||||
@Override |
||||
public Optional<String> taskDependentMsg(long projectCode, long processDefinitionCode, long taskCode) { |
||||
long queryTaskCode = 0; |
||||
if (taskCode != 0) { |
||||
queryTaskCode = taskCode; |
||||
} |
||||
List<ProcessTaskLineage> dependentProcessList = |
||||
processTaskLineageDao.queryWorkFlowLineageByDept(projectCode, processDefinitionCode, queryTaskCode); |
||||
if (CollectionUtils.isEmpty(dependentProcessList)) { |
||||
return Optional.empty(); |
||||
} |
||||
|
||||
List<String> taskDepStrList = new ArrayList<>(); |
||||
|
||||
for (ProcessTaskLineage processTaskLineage : dependentProcessList) { |
||||
ProcessDefinition processDefinition = |
||||
processDefinitionMapper.queryByCode(processTaskLineage.getDeptProcessDefinitionCode()); |
||||
String taskName = ""; |
||||
if (processTaskLineage.getTaskDefinitionCode() != 0) { |
||||
TaskDefinition taskDefinition = |
||||
taskDefinitionMapper.queryByCode(processTaskLineage.getTaskDefinitionCode()); |
||||
taskName = taskDefinition.getName(); |
||||
} |
||||
taskDepStrList.add(String.format(Constants.FORMAT_S_S_COLON, processDefinition.getName(), taskName)); |
||||
} |
||||
|
||||
String taskDepStr = String.join(Constants.COMMA, taskDepStrList); |
||||
if (taskCode != 0) { |
||||
TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCode); |
||||
return Optional |
||||
.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), taskDefinition.getName(), |
||||
taskDepStr)); |
||||
} else { |
||||
return Optional.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), "", |
||||
taskDepStr)); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Query downstream tasks depend on a process definition or a task |
||||
* |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @return downstream dependent process definition list |
||||
*/ |
||||
@Override |
||||
public List<DependentProcessDefinition> queryDownstreamDependentProcessDefinitions(Long processDefinitionCode) { |
||||
List<DependentProcessDefinition> dependentProcessDefinitionList = new ArrayList<>(); |
||||
List<ProcessTaskLineage> processTaskLineageList = |
||||
processTaskLineageDao.queryWorkFlowLineageByDept(Constants.DEFAULT_PROJECT_CODE, processDefinitionCode, |
||||
Constants.DEPENDENT_ALL_TASK); |
||||
if (processTaskLineageList.isEmpty()) { |
||||
return dependentProcessDefinitionList; |
||||
} |
||||
|
||||
List<ProcessDefinition> processDefinitionList = |
||||
processDefinitionMapper.queryByCodes(processTaskLineageList.stream() |
||||
.map(ProcessTaskLineage::getDeptProcessDefinitionCode).distinct().collect(Collectors.toList())); |
||||
List<TaskDefinition> taskDefinitionList = taskDefinitionMapper.queryByCodeList(processTaskLineageList.stream() |
||||
.map(ProcessTaskLineage::getDeptTaskDefinitionCode).distinct().collect(Collectors.toList())); |
||||
for (TaskDefinition taskDefinition : taskDefinitionList) { |
||||
DependentProcessDefinition dependentProcessDefinition = new DependentProcessDefinition(); |
||||
processTaskLineageList.stream() |
||||
.filter(processLineage -> processLineage.getDeptTaskDefinitionCode() == taskDefinition.getCode()) |
||||
.findFirst() |
||||
.ifPresent(processLineage -> { |
||||
dependentProcessDefinition |
||||
.setProcessDefinitionCode(processLineage.getDeptProcessDefinitionCode()); |
||||
dependentProcessDefinition.setTaskDefinitionCode(taskDefinition.getCode()); |
||||
dependentProcessDefinition.setTaskParams(taskDefinition.getTaskParams()); |
||||
dependentProcessDefinition.setWorkerGroup(taskDefinition.getWorkerGroup()); |
||||
}); |
||||
processDefinitionList.stream() |
||||
.filter(processDefinition -> processDefinition.getCode() == dependentProcessDefinition |
||||
.getProcessDefinitionCode()) |
||||
.findFirst() |
||||
.ifPresent(processDefinition -> { |
||||
dependentProcessDefinition.setProcessDefinitionVersion(processDefinition.getVersion()); |
||||
}); |
||||
} |
||||
|
||||
return dependentProcessDefinitionList; |
||||
} |
||||
|
||||
@Override |
||||
public List<DependentLineageTask> queryDependentProcessDefinitions(long projectCode, long processDefinitionCode, |
||||
Long taskCode) { |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
throw new ServiceException(Status.PROJECT_NOT_FOUND, projectCode); |
||||
} |
||||
List<ProcessTaskLineage> processTaskLineageList = processTaskLineageDao.queryWorkFlowLineageByDept(projectCode, |
||||
processDefinitionCode, taskCode == null ? 0 : taskCode); |
||||
List<ProcessDefinition> processDefinitionList = |
||||
processDefinitionMapper.queryByCodes(processTaskLineageList.stream() |
||||
.map(ProcessTaskLineage::getProcessDefinitionCode).distinct().collect(Collectors.toList())); |
||||
List<TaskDefinition> taskDefinitionList = taskDefinitionMapper.queryByCodeList(processTaskLineageList.stream() |
||||
.map(ProcessTaskLineage::getTaskDefinitionCode).filter(code -> code != 0).distinct() |
||||
.collect(Collectors.toList())); |
||||
List<DependentLineageTask> dependentLineageTaskList = new ArrayList<>(); |
||||
for (ProcessTaskLineage processTaskLineage : processTaskLineageList) { |
||||
DependentLineageTask dependentLineageTask = new DependentLineageTask(); |
||||
taskDefinitionList.stream() |
||||
.filter(taskDefinition -> taskDefinition.getCode() == processTaskLineage.getTaskDefinitionCode()) |
||||
.findFirst() |
||||
.ifPresent(taskDefinition -> { |
||||
dependentLineageTask.setTaskDefinitionCode(taskDefinition.getCode()); |
||||
dependentLineageTask.setTaskDefinitionName(taskDefinition.getName()); |
||||
}); |
||||
processDefinitionList.stream() |
||||
.filter(processDefinition -> processDefinition.getCode() == processTaskLineage |
||||
.getProcessDefinitionCode()) |
||||
.findFirst() |
||||
.ifPresent(processDefinition -> { |
||||
dependentLineageTask.setProcessDefinitionCode(processDefinition.getCode()); |
||||
dependentLineageTask.setProcessDefinitionName(processDefinition.getName()); |
||||
dependentLineageTask.setProjectCode(processDefinition.getProjectCode()); |
||||
}); |
||||
dependentLineageTaskList.add(dependentLineageTask); |
||||
} |
||||
return dependentLineageTaskList; |
||||
} |
||||
|
||||
@Override |
||||
public int createProcessLineage(List<ProcessTaskLineage> processTaskLineages) { |
||||
return processTaskLineageDao.batchInsert(processTaskLineages); |
||||
} |
||||
|
||||
@Override |
||||
public int updateProcessLineage(List<ProcessTaskLineage> processTaskLineages) { |
||||
return processTaskLineageDao.updateProcessTaskLineage(processTaskLineages); |
||||
} |
||||
|
||||
@Override |
||||
public int deleteProcessLineage(List<Long> processDefinitionCodes) { |
||||
return processTaskLineageDao.batchDeleteByProcessDefinitionCode(processDefinitionCodes); |
||||
} |
||||
} |
||||
@ -1,306 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.service.impl; |
||||
|
||||
import org.apache.dolphinscheduler.api.enums.Status; |
||||
import org.apache.dolphinscheduler.api.exceptions.ServiceException; |
||||
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; |
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.DependentProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.Project; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskMainInfo; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; |
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; |
||||
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; |
||||
|
||||
import org.apache.commons.lang3.StringUtils; |
||||
|
||||
import java.text.MessageFormat; |
||||
import java.util.ArrayList; |
||||
import java.util.HashMap; |
||||
import java.util.HashSet; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Map.Entry; |
||||
import java.util.Objects; |
||||
import java.util.Optional; |
||||
import java.util.Set; |
||||
import java.util.stream.Collectors; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
import org.springframework.util.CollectionUtils; |
||||
|
||||
/** |
||||
* work flow lineage service impl |
||||
*/ |
||||
@Slf4j |
||||
@Service |
||||
public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkFlowLineageService { |
||||
|
||||
@Autowired |
||||
private WorkFlowLineageMapper workFlowLineageMapper; |
||||
|
||||
@Autowired |
||||
private ProjectMapper projectMapper; |
||||
|
||||
@Autowired |
||||
private TaskDefinitionLogMapper taskDefinitionLogMapper; |
||||
|
||||
@Autowired |
||||
private TaskDefinitionMapper taskDefinitionMapper; |
||||
|
||||
@Override |
||||
public List<WorkFlowLineage> queryWorkFlowLineageByName(long projectCode, String workFlowName) { |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
throw new ServiceException(Status.PROJECT_NOT_FOUND, projectCode); |
||||
} |
||||
return workFlowLineageMapper.queryWorkFlowLineageByName(projectCode, workFlowName); |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, Object> queryWorkFlowLineageByCode(long projectCode, long sourceWorkFlowCode) { |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
throw new ServiceException(Status.PROJECT_NOT_FOUND, projectCode); |
||||
} |
||||
List<WorkFlowLineage> workFlowLineages = new ArrayList<>(); |
||||
Set<WorkFlowRelation> workFlowRelations = new HashSet<>(); |
||||
recursiveWorkFlow(projectCode, sourceWorkFlowCode, workFlowLineages, workFlowRelations); |
||||
Map<String, Object> workFlowLists = new HashMap<>(); |
||||
// todo: use vo
|
||||
workFlowLists.put(Constants.WORKFLOW_LIST, workFlowLineages); |
||||
workFlowLists.put(Constants.WORKFLOW_RELATION_LIST, workFlowRelations); |
||||
return workFlowLists; |
||||
} |
||||
|
||||
private void recursiveWorkFlow(long projectCode, |
||||
long sourceWorkFlowCode, |
||||
List<WorkFlowLineage> workFlowLineages, |
||||
Set<WorkFlowRelation> workFlowRelations) { |
||||
workFlowLineages.add(workFlowLineageMapper.queryWorkFlowLineageByCode(projectCode, sourceWorkFlowCode)); |
||||
|
||||
List<WorkFlowLineage> downStreamWorkFlowLineages = |
||||
workFlowLineageMapper.queryDownstreamLineageByProcessDefinitionCode(sourceWorkFlowCode, "DEPENDENT"); |
||||
workFlowLineages.addAll(downStreamWorkFlowLineages); |
||||
downStreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations |
||||
.add(new WorkFlowRelation(sourceWorkFlowCode, workFlowLineage.getWorkFlowCode()))); |
||||
|
||||
List<WorkFlowLineage> upstreamWorkFlowLineages = new ArrayList<>(); |
||||
getUpstreamLineages(sourceWorkFlowCode, upstreamWorkFlowLineages); |
||||
workFlowLineages.addAll(upstreamWorkFlowLineages); |
||||
upstreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations |
||||
.add(new WorkFlowRelation(workFlowLineage.getWorkFlowCode(), sourceWorkFlowCode))); |
||||
} |
||||
|
||||
private void getUpstreamLineages(long sourceWorkFlowCode, |
||||
List<WorkFlowLineage> upstreamWorkFlowLineages) { |
||||
List<DependentProcessDefinition> workFlowDependentDefinitionList = |
||||
workFlowLineageMapper.queryUpstreamDependentParamsByProcessDefinitionCode(sourceWorkFlowCode, |
||||
"DEPENDENT"); |
||||
|
||||
List<Long> upstreamProcessDefinitionCodes = new ArrayList<>(); |
||||
|
||||
getProcessDefinitionCodeByDependentDefinitionList(workFlowDependentDefinitionList, |
||||
upstreamProcessDefinitionCodes); |
||||
|
||||
if (!upstreamProcessDefinitionCodes.isEmpty()) { |
||||
upstreamWorkFlowLineages.addAll( |
||||
workFlowLineageMapper.queryWorkFlowLineageByProcessDefinitionCodes(upstreamProcessDefinitionCodes)); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* get dependent process definition code by dependent process definition list |
||||
*/ |
||||
private void getProcessDefinitionCodeByDependentDefinitionList(List<DependentProcessDefinition> dependentDefinitionList, |
||||
List<Long> processDefinitionCodes) { |
||||
for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) { |
||||
for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters() |
||||
.getDependence().getDependTaskList()) { |
||||
for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { |
||||
if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) { |
||||
processDefinitionCodes.add(dependentItem.getDefinitionCode()); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public Map<String, Object> queryWorkFlowLineage(long projectCode) { |
||||
Map<String, Object> result = new HashMap<>(); |
||||
Project project = projectMapper.queryByCode(projectCode); |
||||
if (project == null) { |
||||
log.error("Project does not exist, projectCode:{}.", projectCode); |
||||
putMsg(result, Status.PROJECT_NOT_FOUND, projectCode); |
||||
return result; |
||||
} |
||||
List<ProcessLineage> processLineages = workFlowLineageMapper.queryProcessLineage(projectCode); |
||||
Map<Long, WorkFlowLineage> workFlowLineagesMap = new HashMap<>(); |
||||
Set<WorkFlowRelation> workFlowRelations = new HashSet<>(); |
||||
if (!processLineages.isEmpty()) { |
||||
List<WorkFlowLineage> workFlowLineages = |
||||
workFlowLineageMapper.queryWorkFlowLineageByLineage(processLineages); |
||||
workFlowLineagesMap = workFlowLineages.stream() |
||||
.collect(Collectors.toMap(WorkFlowLineage::getWorkFlowCode, workFlowLineage -> workFlowLineage)); |
||||
Map<Long, List<TaskDefinition>> workFlowMap = new HashMap<>(); |
||||
for (ProcessLineage processLineage : processLineages) { |
||||
workFlowMap.compute(processLineage.getProcessDefinitionCode(), (k, v) -> { |
||||
if (v == null) { |
||||
v = new ArrayList<>(); |
||||
} |
||||
if (processLineage.getPreTaskCode() > 0) { |
||||
v.add(new TaskDefinition(processLineage.getPreTaskCode(), processLineage.getPreTaskVersion())); |
||||
} |
||||
if (processLineage.getPostTaskCode() > 0) { |
||||
v.add(new TaskDefinition(processLineage.getPostTaskCode(), |
||||
processLineage.getPostTaskVersion())); |
||||
} |
||||
return v; |
||||
}); |
||||
} |
||||
for (Entry<Long, List<TaskDefinition>> workFlow : workFlowMap.entrySet()) { |
||||
Set<Long> sourceWorkFlowCodes = |
||||
querySourceWorkFlowCodes(projectCode, workFlow.getKey(), workFlow.getValue()); |
||||
if (sourceWorkFlowCodes.isEmpty()) { |
||||
workFlowRelations.add(new WorkFlowRelation(0L, workFlow.getKey())); |
||||
} else { |
||||
workFlowLineagesMap.get(workFlow.getKey()) |
||||
.setSourceWorkFlowCode(StringUtils.join(sourceWorkFlowCodes, Constants.COMMA)); |
||||
sourceWorkFlowCodes |
||||
.forEach(code -> workFlowRelations.add(new WorkFlowRelation(code, workFlow.getKey()))); |
||||
} |
||||
} |
||||
} |
||||
Map<String, Object> workFlowLists = new HashMap<>(); |
||||
workFlowLists.put(Constants.WORKFLOW_LIST, workFlowLineagesMap.values()); |
||||
workFlowLists.put(Constants.WORKFLOW_RELATION_LIST, workFlowRelations); |
||||
result.put(Constants.DATA_LIST, workFlowLists); |
||||
putMsg(result, Status.SUCCESS); |
||||
return result; |
||||
} |
||||
|
||||
private Set<Long> querySourceWorkFlowCodes(long projectCode, long workFlowCode, |
||||
List<TaskDefinition> taskDefinitionList) { |
||||
Set<Long> sourceWorkFlowCodes = new HashSet<>(); |
||||
if (taskDefinitionList == null || taskDefinitionList.isEmpty()) { |
||||
return sourceWorkFlowCodes; |
||||
} |
||||
List<TaskDefinitionLog> taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList); |
||||
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { |
||||
if (taskDefinitionLog.getProjectCode() == projectCode) { |
||||
if (TaskTypeUtils.isDependentTask(taskDefinitionLog.getTaskType())) { |
||||
DependentParameters dependentParameters = |
||||
JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); |
||||
if (dependentParameters != null) { |
||||
List<DependentTaskModel> dependTaskList = |
||||
dependentParameters.getDependence().getDependTaskList(); |
||||
if (!CollectionUtils.isEmpty(dependTaskList)) { |
||||
for (DependentTaskModel taskModel : dependTaskList) { |
||||
List<DependentItem> dependItemList = taskModel.getDependItemList(); |
||||
for (DependentItem dependentItem : dependItemList) { |
||||
if (dependentItem.getProjectCode() == projectCode |
||||
&& dependentItem.getDefinitionCode() != workFlowCode) { |
||||
sourceWorkFlowCodes.add(dependentItem.getDefinitionCode()); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
return sourceWorkFlowCodes; |
||||
} |
||||
|
||||
/** |
||||
* Query and return tasks dependence with string format, is a wrapper of queryTaskDepOnTask and task query method. |
||||
* |
||||
* @param projectCode Project code want to query tasks dependence |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @param taskCode Task code want to query tasks dependence |
||||
* @return Optional of formatter message |
||||
*/ |
||||
@Override |
||||
public Optional<String> taskDepOnTaskMsg(long projectCode, long processDefinitionCode, long taskCode) { |
||||
List<TaskMainInfo> tasksDep = |
||||
workFlowLineageMapper.queryTaskDepOnTask(projectCode, processDefinitionCode, taskCode); |
||||
if (CollectionUtils.isEmpty(tasksDep)) { |
||||
return Optional.empty(); |
||||
} |
||||
|
||||
String taskDepStr = tasksDep.stream().map( |
||||
task -> String.format(Constants.FORMAT_S_S_COLON, task.getProcessDefinitionName(), task.getTaskName())) |
||||
.collect(Collectors.joining(Constants.COMMA)); |
||||
TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCode); |
||||
return Optional.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), taskDefinition.getName(), |
||||
taskDepStr)); |
||||
} |
||||
|
||||
/** |
||||
* Query tasks depend on process definition, include upstream or downstream |
||||
* |
||||
* @param projectCode Project code want to query tasks dependence |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @return Set of TaskMainInfo |
||||
*/ |
||||
@Override |
||||
public Set<TaskMainInfo> queryTaskDepOnProcess(long projectCode, long processDefinitionCode) { |
||||
Set<TaskMainInfo> taskMainInfos = new HashSet<>(); |
||||
List<TaskMainInfo> taskDependents = |
||||
workFlowLineageMapper.queryTaskDependentOnProcess(processDefinitionCode, 0); |
||||
List<TaskMainInfo> taskSubProcess = |
||||
workFlowLineageMapper.queryTaskSubProcessDepOnProcess(projectCode, processDefinitionCode); |
||||
taskMainInfos.addAll(taskDependents); |
||||
taskMainInfos.addAll(taskSubProcess); |
||||
return taskMainInfos; |
||||
} |
||||
|
||||
/** |
||||
* Query downstream tasks depend on a process definition or a task |
||||
* |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @param taskCode Task code want to query tasks dependence |
||||
* @return downstream dependent tasks |
||||
*/ |
||||
@Override |
||||
public Map<String, Object> queryDownstreamDependentTasks(Long processDefinitionCode, Long taskCode) { |
||||
Map<String, Object> result = new HashMap<>(); |
||||
List<TaskMainInfo> taskDependents = |
||||
workFlowLineageMapper.queryTaskDependentOnProcess(processDefinitionCode, |
||||
Objects.isNull(taskCode) ? 0 : taskCode.longValue()); |
||||
result.put(Constants.DATA_LIST, taskDependents); |
||||
putMsg(result, Status.SUCCESS); |
||||
return result; |
||||
} |
||||
} |
||||
@ -0,0 +1,34 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.entity; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
|
||||
@Data |
||||
@NoArgsConstructor |
||||
@AllArgsConstructor |
||||
public class DependentLineageTask { |
||||
|
||||
private long projectCode; |
||||
private long processDefinitionCode; |
||||
private String processDefinitionName; |
||||
private long taskDefinitionCode; |
||||
private String taskDefinitionName; |
||||
} |
||||
@ -1,115 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.entity; |
||||
|
||||
/** |
||||
* Process lineage |
||||
*/ |
||||
public class ProcessLineage { |
||||
|
||||
/** |
||||
* project code |
||||
*/ |
||||
private long projectCode; |
||||
|
||||
/** |
||||
* post task code |
||||
*/ |
||||
private long postTaskCode; |
||||
|
||||
/** |
||||
* post task version |
||||
*/ |
||||
private int postTaskVersion; |
||||
|
||||
/** |
||||
* pre task code |
||||
*/ |
||||
private long preTaskCode; |
||||
|
||||
/** |
||||
* pre task version |
||||
*/ |
||||
private int preTaskVersion; |
||||
|
||||
/** |
||||
* process definition code |
||||
*/ |
||||
private long processDefinitionCode; |
||||
|
||||
/** |
||||
* process definition version |
||||
*/ |
||||
private int processDefinitionVersion; |
||||
|
||||
public long getProjectCode() { |
||||
return projectCode; |
||||
} |
||||
|
||||
public void setProjectCode(long projectCode) { |
||||
this.projectCode = projectCode; |
||||
} |
||||
|
||||
public long getPostTaskCode() { |
||||
return postTaskCode; |
||||
} |
||||
|
||||
public void setPostTaskCode(long postTaskCode) { |
||||
this.postTaskCode = postTaskCode; |
||||
} |
||||
|
||||
public int getPostTaskVersion() { |
||||
return postTaskVersion; |
||||
} |
||||
|
||||
public void setPostTaskVersion(int postTaskVersion) { |
||||
this.postTaskVersion = postTaskVersion; |
||||
} |
||||
|
||||
public long getPreTaskCode() { |
||||
return preTaskCode; |
||||
} |
||||
|
||||
public void setPreTaskCode(long preTaskCode) { |
||||
this.preTaskCode = preTaskCode; |
||||
} |
||||
|
||||
public int getPreTaskVersion() { |
||||
return preTaskVersion; |
||||
} |
||||
|
||||
public void setPreTaskVersion(int preTaskVersion) { |
||||
this.preTaskVersion = preTaskVersion; |
||||
} |
||||
|
||||
public long getProcessDefinitionCode() { |
||||
return processDefinitionCode; |
||||
} |
||||
|
||||
public void setProcessDefinitionCode(long processDefinitionCode) { |
||||
this.processDefinitionCode = processDefinitionCode; |
||||
} |
||||
|
||||
public int getProcessDefinitionVersion() { |
||||
return processDefinitionVersion; |
||||
} |
||||
|
||||
public void setProcessDefinitionVersion(int processDefinitionVersion) { |
||||
this.processDefinitionVersion = processDefinitionVersion; |
||||
} |
||||
} |
||||
@ -0,0 +1,48 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.entity; |
||||
|
||||
import java.util.Date; |
||||
|
||||
import lombok.AllArgsConstructor; |
||||
import lombok.Data; |
||||
import lombok.NoArgsConstructor; |
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType; |
||||
import com.baomidou.mybatisplus.annotation.TableId; |
||||
import com.baomidou.mybatisplus.annotation.TableName; |
||||
|
||||
@Data |
||||
@NoArgsConstructor |
||||
@AllArgsConstructor |
||||
@TableName("t_ds_process_task_lineage") |
||||
public class ProcessTaskLineage { |
||||
|
||||
@TableId(value = "id", type = IdType.AUTO) |
||||
private Integer id; |
||||
|
||||
private long processDefinitionCode; |
||||
private int processDefinitionVersion; |
||||
private long taskDefinitionCode; |
||||
private int taskDefinitionVersion; |
||||
private long deptProjectCode; |
||||
private long deptProcessDefinitionCode; |
||||
private long deptTaskDefinitionCode; |
||||
private Date createTime; |
||||
private Date updateTime; |
||||
} |
||||
@ -0,0 +1,36 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.entity; |
||||
|
||||
import java.util.Date; |
||||
|
||||
import lombok.Data; |
||||
|
||||
@Data |
||||
public class WorkFlowRelationDetail { |
||||
|
||||
private long workFlowCode; |
||||
private String workFlowName; |
||||
private String workFlowPublishStatus; |
||||
private Date scheduleStartTime; |
||||
private Date scheduleEndTime; |
||||
private String crontab; |
||||
private int schedulePublishStatus; |
||||
private String sourceWorkFlowCode; |
||||
|
||||
} |
||||
@ -0,0 +1,48 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.mapper; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail; |
||||
|
||||
import org.apache.ibatis.annotations.Param; |
||||
|
||||
import java.util.List; |
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; |
||||
|
||||
public interface ProcessTaskLineageMapper extends BaseMapper<ProcessTaskLineage> { |
||||
|
||||
int batchDeleteByProcessDefinitionCode(@Param("processDefinitionCodes") List<Long> processDefinitionCodes); |
||||
|
||||
int batchInsert(@Param("processLineages") List<ProcessTaskLineage> processTaskLineages); |
||||
|
||||
List<ProcessTaskLineage> queryByProjectCode(@Param("projectCode") long projectCode); |
||||
|
||||
List<WorkFlowRelationDetail> queryWorkFlowLineageByCode(@Param("processDefinitionCode") long processDefinitionCode); |
||||
|
||||
List<WorkFlowRelationDetail> queryWorkFlowLineageByName(@Param("projectCode") long projectCode, |
||||
@Param("processDefinitionName") String processDefinitionName); |
||||
|
||||
List<ProcessTaskLineage> queryWorkFlowLineageByDept(@Param("deptProjectCode") long deptProjectCode, |
||||
@Param("deptProcessDefinitionCode") long deptProcessDefinitionCode, |
||||
@Param("deptTaskDefinitionCode") long deptTaskDefinitionCode); |
||||
|
||||
List<ProcessTaskLineage> queryByProcessDefinitionCode(@Param("processDefinitionCode") long processDefinitionCode); |
||||
|
||||
} |
||||
@ -1,148 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.mapper; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.DependentProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskMainInfo; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; |
||||
|
||||
import org.apache.ibatis.annotations.Param; |
||||
|
||||
import java.util.List; |
||||
|
||||
public interface WorkFlowLineageMapper { |
||||
|
||||
/** |
||||
* queryByName |
||||
* |
||||
* @param projectCode projectCode |
||||
* @param workFlowName workFlowName |
||||
* @return WorkFlowLineage list |
||||
*/ |
||||
List<WorkFlowLineage> queryWorkFlowLineageByName(@Param("projectCode") long projectCode, |
||||
@Param("workFlowName") String workFlowName); |
||||
|
||||
/** |
||||
* queryWorkFlowLineageByCode |
||||
* |
||||
* @param projectCode projectCode |
||||
* @param workFlowCode workFlowCode |
||||
* @return WorkFlowLineage |
||||
*/ |
||||
WorkFlowLineage queryWorkFlowLineageByCode(@Param("projectCode") long projectCode, |
||||
@Param("workFlowCode") long workFlowCode); |
||||
|
||||
/** |
||||
* queryWorkFlowLineageByProcessDefinitionCodes |
||||
* |
||||
* @param workFlowCodes workFlowCodes |
||||
* @return WorkFlowLineage |
||||
*/ |
||||
List<WorkFlowLineage> queryWorkFlowLineageByProcessDefinitionCodes(@Param("workFlowCodes") List<Long> workFlowCodes); |
||||
|
||||
/** |
||||
* queryWorkFlowLineageByCode |
||||
* |
||||
* @param processLineages processLineages |
||||
* @return WorkFlowLineage list |
||||
*/ |
||||
List<WorkFlowLineage> queryWorkFlowLineageByLineage(@Param("processLineages") List<ProcessLineage> processLineages); |
||||
|
||||
/** |
||||
* queryProcessLineage |
||||
* |
||||
* @param projectCode projectCode |
||||
* @return ProcessLineage list |
||||
*/ |
||||
List<ProcessLineage> queryProcessLineage(@Param("projectCode") long projectCode); |
||||
|
||||
/** |
||||
* queryCodeRelation |
||||
* |
||||
* @param projectCode projectCode |
||||
* @param processDefinitionCode processDefinitionCode |
||||
* @return ProcessLineage list |
||||
*/ |
||||
List<ProcessLineage> queryProcessLineageByCode(@Param("projectCode") long projectCode, |
||||
@Param("processDefinitionCode") long processDefinitionCode); |
||||
|
||||
/** |
||||
* query process definition by name |
||||
* |
||||
* @return dependent process definition |
||||
*/ |
||||
List<DependentProcessDefinition> queryDependentProcessDefinitionByProcessDefinitionCode(@Param("code") long code); |
||||
|
||||
/** |
||||
* query downstream work flow lineage by process definition code |
||||
* |
||||
* @return dependent process definition |
||||
*/ |
||||
List<WorkFlowLineage> queryDownstreamLineageByProcessDefinitionCode(@Param("code") long code, |
||||
@Param("taskType") String taskType); |
||||
|
||||
/** |
||||
* query upstream work flow dependent task params by process definition code |
||||
* |
||||
* @return task_params |
||||
*/ |
||||
List<DependentProcessDefinition> queryUpstreamDependentParamsByProcessDefinitionCode(@Param("code") long code, |
||||
@Param("taskType") String taskType); |
||||
|
||||
/** |
||||
* Query all tasks type sub process depend on process definition. |
||||
* |
||||
* Query all upstream tasks from task type sub process. |
||||
* |
||||
* @param projectCode Project code want to query tasks dependence |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @return List of TaskMainInfo |
||||
*/ |
||||
List<TaskMainInfo> queryTaskSubProcessDepOnProcess(@Param("projectCode") long projectCode, |
||||
@Param("processDefinitionCode") long processDefinitionCode); |
||||
|
||||
/** |
||||
* Query all tasks type dependent depend on process definition. |
||||
* |
||||
* Query all downstream tasks from task type dependent, method `queryTaskDepOnTask` is a proper subset of |
||||
* current method `queryTaskDepOnProcess`. Which mean with the same parameter processDefinitionCode, all tasks in |
||||
* `queryTaskDepOnTask` are in the result of method `queryTaskDepOnProcess`. |
||||
* |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @param taskCode Task code want to query tasks dependence |
||||
* @return List of TaskMainInfo |
||||
*/ |
||||
List<TaskMainInfo> queryTaskDependentOnProcess(@Param("processDefinitionCode") long processDefinitionCode, |
||||
@Param("taskCode") long taskCode); |
||||
|
||||
/** |
||||
* Query all tasks depend on task, only downstream task support currently(from dependent task type). |
||||
* |
||||
* In case of dependent task type, method `queryTaskDepOnTask` is a proper subset of `queryTaskDepOnProcess`. Which |
||||
* mean with the same processDefinitionCode, all tasks in `queryTaskDepOnTask` are in method `queryTaskDepOnProcess`. |
||||
* |
||||
* @param projectCode Project code want to query tasks dependence |
||||
* @param processDefinitionCode Process definition code want to query tasks dependence |
||||
* @param taskCode Task code want to query tasks dependence |
||||
* @return dependent process definition |
||||
*/ |
||||
List<TaskMainInfo> queryTaskDepOnTask(@Param("projectCode") long projectCode, |
||||
@Param("processDefinitionCode") long processDefinitionCode, |
||||
@Param("taskCode") long taskCode); |
||||
} |
||||
@ -0,0 +1,45 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.repository; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail; |
||||
|
||||
import java.util.List; |
||||
|
||||
public interface ProcessTaskLineageDao extends IDao<ProcessTaskLineage> { |
||||
|
||||
int batchDeleteByProcessDefinitionCode(List<Long> processDefinitionCodes); |
||||
|
||||
int batchInsert(List<ProcessTaskLineage> processTaskLineages); |
||||
|
||||
List<ProcessTaskLineage> queryByProjectCode(long projectCode); |
||||
|
||||
List<WorkFlowRelationDetail> queryWorkFlowLineageByCode(long processDefinitionCode); |
||||
|
||||
List<WorkFlowRelationDetail> queryWorkFlowLineageByName(long projectCode, |
||||
String processDefinitionName); |
||||
|
||||
List<ProcessTaskLineage> queryWorkFlowLineageByDept(long deptProjectCode, |
||||
long deptProcessDefinitionCode, |
||||
long deptTaskDefinitionCode); |
||||
|
||||
List<ProcessTaskLineage> queryByProcessDefinitionCode(long processDefinitionCode); |
||||
|
||||
int updateProcessTaskLineage(List<ProcessTaskLineage> processTaskLineages); |
||||
} |
||||
@ -0,0 +1,97 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.repository.impl; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskLineageMapper; |
||||
import org.apache.dolphinscheduler.dao.repository.BaseDao; |
||||
import org.apache.dolphinscheduler.dao.repository.ProcessTaskLineageDao; |
||||
|
||||
import org.apache.commons.collections4.CollectionUtils; |
||||
|
||||
import java.util.List; |
||||
import java.util.stream.Collectors; |
||||
|
||||
import lombok.NonNull; |
||||
|
||||
import org.springframework.stereotype.Repository; |
||||
|
||||
@Repository |
||||
public class ProcessTaskLineageDaoImpl extends BaseDao<ProcessTaskLineage, ProcessTaskLineageMapper> |
||||
implements |
||||
ProcessTaskLineageDao { |
||||
|
||||
public ProcessTaskLineageDaoImpl(@NonNull ProcessTaskLineageMapper processTaskLineageMapper) { |
||||
super(processTaskLineageMapper); |
||||
} |
||||
|
||||
@Override |
||||
public int batchDeleteByProcessDefinitionCode(List<Long> processDefinitionCodes) { |
||||
if (CollectionUtils.isEmpty(processDefinitionCodes)) { |
||||
return 0; |
||||
} |
||||
return mybatisMapper.batchDeleteByProcessDefinitionCode(processDefinitionCodes); |
||||
} |
||||
|
||||
@Override |
||||
public int batchInsert(List<ProcessTaskLineage> processTaskLineages) { |
||||
if (CollectionUtils.isEmpty(processTaskLineages)) { |
||||
return 0; |
||||
} |
||||
return mybatisMapper.batchInsert(processTaskLineages); |
||||
} |
||||
|
||||
@Override |
||||
public List<ProcessTaskLineage> queryByProjectCode(long projectCode) { |
||||
return mybatisMapper.queryByProjectCode(projectCode); |
||||
} |
||||
|
||||
@Override |
||||
public List<WorkFlowRelationDetail> queryWorkFlowLineageByCode(long processDefinitionCode) { |
||||
return mybatisMapper.queryWorkFlowLineageByCode(processDefinitionCode); |
||||
} |
||||
|
||||
@Override |
||||
public List<WorkFlowRelationDetail> queryWorkFlowLineageByName(long projectCode, String processDefinitionName) { |
||||
return mybatisMapper.queryWorkFlowLineageByName(projectCode, processDefinitionName); |
||||
} |
||||
|
||||
@Override |
||||
public List<ProcessTaskLineage> queryWorkFlowLineageByDept(long deptProjectCode, long deptProcessDefinitionCode, |
||||
long deptTaskDefinitionCode) { |
||||
return mybatisMapper.queryWorkFlowLineageByDept(deptProjectCode, deptProcessDefinitionCode, |
||||
deptTaskDefinitionCode); |
||||
} |
||||
|
||||
@Override |
||||
public List<ProcessTaskLineage> queryByProcessDefinitionCode(long processDefinitionCode) { |
||||
return mybatisMapper.queryByProcessDefinitionCode(processDefinitionCode); |
||||
} |
||||
|
||||
@Override |
||||
public int updateProcessTaskLineage(List<ProcessTaskLineage> processTaskLineages) { |
||||
if (CollectionUtils.isEmpty(processTaskLineages)) { |
||||
return 0; |
||||
} |
||||
this.batchDeleteByProcessDefinitionCode( |
||||
processTaskLineages.stream().map(ProcessTaskLineage::getProcessDefinitionCode) |
||||
.distinct().collect(Collectors.toList())); |
||||
return mybatisMapper.batchInsert(processTaskLineages); |
||||
} |
||||
} |
||||
@ -0,0 +1,133 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > |
||||
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ProcessTaskLineageMapper"> |
||||
<sql id="baseSql"> |
||||
id |
||||
, process_definition_code |
||||
, process_definition_version |
||||
, task_definition_code |
||||
, task_definition_version |
||||
, dept_project_code |
||||
, dept_process_definition_code |
||||
, dept_task_definition_code |
||||
, create_time |
||||
, update_time |
||||
</sql> |
||||
|
||||
<delete id="batchDeleteByProcessDefinitionCode"> |
||||
delete from t_ds_process_task_lineage |
||||
where process_definition_code in |
||||
<foreach collection="processDefinitionCodes" index="index" item="i" open="(" separator="," close=")"> |
||||
#{i} |
||||
</foreach> |
||||
</delete> |
||||
|
||||
<insert id="batchInsert"> |
||||
insert into t_ds_process_task_lineage (process_definition_code, process_definition_version, task_definition_code, |
||||
task_definition_version, dept_project_code, dept_process_definition_code, dept_task_definition_code) |
||||
values |
||||
<foreach collection="processTaskLineages" item="processTaskLineage" separator=","> |
||||
(#{processTaskLineage.processDefinitionCode},#{processTaskLineage.processDefinitionVersion}, |
||||
#{processTaskLineage.taskDefinitionCode},#{processTaskLineage.taskDefinitionVersion}, |
||||
#{processTaskLineage.deptProjectCode},#{processTaskLineage.deptProcessDefinitionCode}, |
||||
#{processTaskLineage.deptTaskDefinitionCode}) |
||||
</foreach> |
||||
</insert> |
||||
|
||||
<select id="queryByProjectCode" resultType="org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage"> |
||||
select |
||||
<include refid="baseSql"/> |
||||
from |
||||
t_ds_process_task_lineage |
||||
where process_definition_code in (select code from t_ds_process_definition where project_code = #{projectCode}) |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByCode" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail"> |
||||
select |
||||
pd.code as work_flow_code |
||||
,pd.name as work_flow_name |
||||
,pd.release_state as work_flow_publish_status |
||||
,schd.start_time as schedule_start_time |
||||
,schd.end_time as schedule_end_time |
||||
,schd.crontab as crontab |
||||
,schd.release_state as schedule_publish_status |
||||
from t_ds_process_definition pd |
||||
left join t_ds_schedules schd on pd.code = schd.process_definition_code |
||||
where pd.code = #{processDefinitionCode} |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByName" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail"> |
||||
select |
||||
pd.code as work_flow_code |
||||
,pd.name as work_flow_name |
||||
,pd.release_state as work_flow_publish_status |
||||
,schd.start_time as schedule_start_time |
||||
,schd.end_time as schedule_end_time |
||||
,schd.crontab as crontab |
||||
,schd.release_state as schedule_publish_status |
||||
from t_ds_process_definition pd |
||||
left join t_ds_schedules schd on pd.code = schd.process_definition_code |
||||
where 1=1 |
||||
<if test="processDefinitionName != null and processDefinitionName != ''"> |
||||
and pd.name = #{processDefinitionName} |
||||
</if> |
||||
and pd.project_code = #{projectCode} |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByCode" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowRelationDetail"> |
||||
select |
||||
pd.code as work_flow_code |
||||
,pd.name as work_flow_name |
||||
,pd.release_state as work_flow_publish_status |
||||
,schd.start_time as schedule_start_time |
||||
,schd.end_time as schedule_end_time |
||||
,schd.crontab as crontab |
||||
,schd.release_state as schedule_publish_status |
||||
from t_ds_process_definition pd |
||||
left join t_ds_schedules schd on pd.code = schd.process_definition_code |
||||
where 1=1 |
||||
and pd.code = #{processDefinitionCode} |
||||
and pd.project_code = #{projectCode} |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByDept" resultType="org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage"> |
||||
select |
||||
<include refid="baseSql"/> |
||||
from |
||||
t_ds_process_task_lineage |
||||
where 1=1 |
||||
<if test="deptProjectCode != null and deptProjectCode != 0"> |
||||
and dept_project_code = #{deptProjectCode} |
||||
</if> |
||||
and dept_process_definition_code = #{deptProcessDefinitionCode} |
||||
<if test="deptTaskDefinitionCode != null and deptTaskDefinitionCode != 0"> |
||||
and dept_task_definition_code = #{deptTaskDefinitionCode} |
||||
</if> |
||||
</select> |
||||
|
||||
<select id="queryByProcessDefinitionCode" resultType="org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage"> |
||||
select |
||||
<include refid="baseSql"/> |
||||
from |
||||
t_ds_process_task_lineage |
||||
where process_definition_code = #{processDefinitionCode} |
||||
</select> |
||||
|
||||
</mapper> |
||||
@ -1,249 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > |
||||
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper"> |
||||
|
||||
<select id="queryWorkFlowLineageByName" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage"> |
||||
select tepd.code as work_flow_code,tepd.name as work_flow_name |
||||
from t_ds_process_definition tepd |
||||
left join t_ds_schedules tes on tepd.code = tes.process_definition_code |
||||
where tepd.project_code = #{projectCode} |
||||
<if test="workFlowName != null and workFlowName != ''"> |
||||
and tepd.name like concat('%', #{workFlowName}, '%') |
||||
</if> |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByCode" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage"> |
||||
select tepd.code as work_flow_code,tepd.name as work_flow_name, |
||||
'' as source_work_flow_code, |
||||
tepd.release_state as work_flow_publish_status, |
||||
tes.start_time as schedule_start_time, |
||||
tes.end_time as schedule_end_time, |
||||
tes.crontab as crontab, |
||||
tes.release_state as schedule_publish_status |
||||
from t_ds_process_definition tepd |
||||
left join t_ds_schedules tes on tepd.code = tes.process_definition_code |
||||
where tepd.project_code = #{projectCode} and tepd.code = #{workFlowCode} |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByProcessDefinitionCodes" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage"> |
||||
select tepd.code as work_flow_code, |
||||
tepd.name as work_flow_name, |
||||
tepd.release_state as work_flow_publish_status, |
||||
tes.start_time as schedule_start_time, |
||||
tes.end_time as schedule_end_time, |
||||
tes.crontab as crontab, |
||||
tes.release_state as schedule_publish_status |
||||
from t_ds_process_definition tepd |
||||
left join t_ds_schedules tes on tepd.code = tes.process_definition_code |
||||
where 1=1 |
||||
<if test="workFlowCodes != null and workFlowCodes.size() != 0 "> |
||||
and tepd.code in |
||||
<foreach collection="workFlowCodes" item="code" index="index" open="(" close=")" separator=","> |
||||
#{code} |
||||
</foreach> |
||||
</if> |
||||
</select> |
||||
|
||||
<select id="queryWorkFlowLineageByLineage" resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage"> |
||||
select tepd.code as work_flow_code,tepd.name as work_flow_name, |
||||
'' as source_work_flow_code, |
||||
tepd.release_state as work_flow_publish_status, |
||||
tes.start_time as schedule_start_time, |
||||
tes.end_time as schedule_end_time, |
||||
tes.crontab as crontab, |
||||
tes.release_state as schedule_publish_status |
||||
from t_ds_process_definition tepd |
||||
left join t_ds_schedules tes on tepd.code = tes.process_definition_code |
||||
where 1=1 |
||||
<if test="processLineages != null and processLineages.size != 0"> |
||||
and |
||||
<foreach collection="processLineages" index="index" item="item" open="(" separator=" or " close=")"> |
||||
(tepd.project_code = #{item.projectCode} |
||||
and tepd.code = #{item.processDefinitionCode}) |
||||
</foreach> |
||||
</if> |
||||
</select> |
||||
|
||||
<select id="queryProcessLineage" resultType="org.apache.dolphinscheduler.dao.entity.ProcessLineage"> |
||||
select ptr.project_code, |
||||
ptr.post_task_code, |
||||
ptr.post_task_version, |
||||
ptr.pre_task_code, |
||||
ptr.pre_task_version, |
||||
ptr.process_definition_code, |
||||
ptr.process_definition_version |
||||
from t_ds_process_definition pd |
||||
join t_ds_process_task_relation ptr on pd.code = ptr.process_definition_code |
||||
and pd.version = ptr.process_definition_version |
||||
where pd.project_code = #{projectCode} |
||||
</select> |
||||
|
||||
<select id="queryProcessLineageByCode" resultType="org.apache.dolphinscheduler.dao.entity.ProcessLineage"> |
||||
select project_code, |
||||
post_task_code, |
||||
post_task_version, |
||||
pre_task_code, |
||||
pre_task_version, |
||||
process_definition_code, |
||||
process_definition_version |
||||
from t_ds_process_task_relation |
||||
where project_code = #{projectCode} |
||||
and process_definition_code = #{processDefinitionCode} |
||||
</select> |
||||
|
||||
<select id="queryUpstreamDependentParamsByProcessDefinitionCode" |
||||
resultType="org.apache.dolphinscheduler.dao.entity.DependentProcessDefinition"> |
||||
SELECT |
||||
DISTINCT c.task_params |
||||
FROM |
||||
t_ds_process_definition a |
||||
JOIN t_ds_process_task_relation b ON a.code = b.process_definition_code AND a.version = b.process_definition_version AND a.project_code = b.project_code |
||||
JOIN t_ds_task_definition c ON c.code = b.pre_task_code and c.version = b.pre_task_version |
||||
WHERE 1=1 |
||||
AND a.code = #{code} |
||||
AND c.task_type = #{taskType} |
||||
; |
||||
</select> |
||||
|
||||
<select id="queryDownstreamLineageByProcessDefinitionCode" |
||||
resultType="org.apache.dolphinscheduler.dao.entity.WorkFlowLineage"> |
||||
SELECT |
||||
c.code AS work_flow_code |
||||
,c.name AS work_flow_name |
||||
,c.release_state AS work_flow_publish_status |
||||
,d.start_time AS schedule_start_time |
||||
,d.end_time AS schedule_end_time |
||||
,d.crontab AS crontab |
||||
,d.release_state AS schedule_publish_status |
||||
,'' AS source_work_flow_code |
||||
FROM |
||||
t_ds_task_definition a |
||||
JOIN t_ds_process_task_relation b ON a.code = b.pre_task_code and a.version = b.pre_task_version |
||||
JOIN t_ds_process_definition c ON c.code = b.process_definition_code AND c.version = b.process_definition_version AND c.project_code = b.project_code |
||||
LEFT JOIN t_ds_schedules d ON d.process_definition_code = c.code |
||||
WHERE 1=1 |
||||
<if test="code != null and code != ''"> |
||||
AND a.task_params LIKE concat('%', #{code}, '%') |
||||
</if> |
||||
AND a.task_type = #{taskType} |
||||
</select> |
||||
|
||||
<select id="queryDependentProcessDefinitionByProcessDefinitionCode" resultType="org.apache.dolphinscheduler.dao.entity.DependentProcessDefinition"> |
||||
SELECT |
||||
c.code AS process_definition_code |
||||
,c.name AS process_definition_name |
||||
,c.version as process_definition_version |
||||
,a.code AS task_definition_code |
||||
,a.task_params |
||||
FROM |
||||
t_ds_task_definition a |
||||
JOIN t_ds_process_task_relation b ON a.code = b.pre_task_code and a.version = b.pre_task_version |
||||
JOIN t_ds_process_definition c ON c.code = b.process_definition_code AND c.version = b.process_definition_version AND c.project_code = b.project_code |
||||
WHERE 1=1 |
||||
<if test="code != null and code != ''"> |
||||
AND a.task_params LIKE concat('%', #{code}, '%') |
||||
</if> |
||||
AND a.task_type = 'DEPENDENT' |
||||
AND c.release_state = 1 |
||||
</select> |
||||
|
||||
<select id="queryTaskSubProcessDepOnProcess" resultType="org.apache.dolphinscheduler.dao.entity.TaskMainInfo"> |
||||
select td.id |
||||
, td.name as taskName |
||||
, td.code as taskCode |
||||
, td.version as taskVersion |
||||
, td.task_type as taskType |
||||
, ptr.process_definition_code as processDefinitionCode |
||||
, pd.name as processDefinitionName |
||||
, pd.version as processDefinitionVersion |
||||
, pd.release_state as processReleaseState |
||||
from t_ds_task_definition td |
||||
join t_ds_process_task_relation ptr on ptr.post_task_code = td.code and td.version = ptr.post_task_version |
||||
join t_ds_process_definition pd on pd.code = ptr.process_definition_code and pd.version = ptr.process_definition_version |
||||
<where> |
||||
<if test="projectCode != 0"> |
||||
and ptr.project_code = #{projectCode} |
||||
</if> |
||||
<!-- ptr.process_definition_code != #{processDefinitionCode} query task not in current workflow --> |
||||
<!-- For subprocess task type, using `concat('%"processDefinitionCode":', #{processDefinitionCode}, '%')` --> |
||||
<if test="processDefinitionCode != 0"> |
||||
and td.task_type = 'SUB_PROCESS' |
||||
and ptr.process_definition_code != #{processDefinitionCode} |
||||
and td.task_params like concat('%"processDefinitionCode":', #{processDefinitionCode}, '%') |
||||
</if> |
||||
</where> |
||||
</select> |
||||
|
||||
<select id="queryTaskDependentOnProcess" resultType="org.apache.dolphinscheduler.dao.entity.TaskMainInfo"> |
||||
select td.id |
||||
, td.name as taskName |
||||
, td.code as taskCode |
||||
, td.version as taskVersion |
||||
, td.task_type as taskType |
||||
, pd.project_code as projectCode |
||||
, ptr.process_definition_code as processDefinitionCode |
||||
, pd.name as processDefinitionName |
||||
, pd.version as processDefinitionVersion |
||||
, pd.release_state as processReleaseState |
||||
from t_ds_task_definition td |
||||
join t_ds_process_task_relation ptr on ptr.post_task_code = td.code and td.version = ptr.post_task_version |
||||
join t_ds_process_definition pd on pd.code = ptr.process_definition_code and pd.version = ptr.process_definition_version |
||||
<where> |
||||
<!-- ptr.process_definition_code != #{processDefinitionCode} query task not in current workflow --> |
||||
<!-- For dependnet task type, using `like concat('%"definitionCode":', #{processDefinitionCode}, '%')` --> |
||||
<if test="processDefinitionCode != 0"> |
||||
and td.task_type = 'DEPENDENT' |
||||
and ptr.process_definition_code != #{processDefinitionCode} |
||||
and td.task_params like concat('%"definitionCode":', #{processDefinitionCode}, '%') |
||||
</if> |
||||
<if test="taskCode != 0"> |
||||
and (td.task_params like concat('%"depTaskCode":', #{taskCode}, '%') or td.task_params like concat('%"depTaskCode":-1%')) |
||||
</if> |
||||
</where> |
||||
</select> |
||||
|
||||
<select id="queryTaskDepOnTask" resultType="org.apache.dolphinscheduler.dao.entity.TaskMainInfo"> |
||||
select td.id |
||||
, td.name as taskName |
||||
, td.code as taskCode |
||||
, td.version as taskVersion |
||||
, td.task_type as taskType |
||||
, ptr.process_definition_code as processDefinitionCode |
||||
, pd.name as processDefinitionName |
||||
, pd.version as processDefinitionVersion |
||||
, pd.release_state as processReleaseState |
||||
from t_ds_task_definition td |
||||
join t_ds_process_task_relation ptr on ptr.post_task_code = td.code and td.version = ptr.post_task_version |
||||
join t_ds_process_definition pd on pd.code = ptr.process_definition_code and pd.version = ptr.process_definition_version |
||||
<where> |
||||
<if test="projectCode != 0"> |
||||
and ptr.project_code = #{projectCode} |
||||
</if> |
||||
<!-- ptr.process_definition_code != #{processDefinitionCode} query task not in current workflow --> |
||||
<if test="processDefinitionCode != 0"> |
||||
and ptr.process_definition_code != #{processDefinitionCode} |
||||
and td.task_params like concat('%"definitionCode":', #{processDefinitionCode}, '%') |
||||
</if> |
||||
<if test="taskCode != 0"> |
||||
and td.task_params like concat('%"depTaskCode":', #{taskCode}, '%') |
||||
</if> |
||||
</where> |
||||
</select> |
||||
</mapper> |
||||
@ -0,0 +1,35 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
DROP TABLE IF EXISTS `t_ds_process_task_lineage`; |
||||
CREATE TABLE `t_ds_process_task_lineage` |
||||
( |
||||
`id` int NOT NULL AUTO_INCREMENT, |
||||
`process_definition_code` bigint NOT NULL DEFAULT 0, |
||||
`process_definition_version` int NOT NULL DEFAULT 0, |
||||
`task_definition_code` bigint NOT NULL DEFAULT 0, |
||||
`task_definition_version` int NOT NULL DEFAULT 0, |
||||
`dept_project_code` bigint NOT NULL DEFAULT 0 COMMENT 'dependent project code', |
||||
`dept_process_definition_code` bigint NOT NULL DEFAULT 0 COMMENT 'dependent process definition code', |
||||
`dept_task_definition_code` bigint NOT NULL DEFAULT 0 COMMENT 'dependent task definition code', |
||||
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', |
||||
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', |
||||
PRIMARY KEY (`id`), |
||||
KEY `idx_process_code_version` (`process_definition_code`,`process_definition_version`), |
||||
KEY `idx_task_code_version` (`task_definition_code`,`task_definition_version`), |
||||
KEY `idx_dept_code` (`dept_project_code`,`dept_process_definition_code`,`dept_task_definition_code`) |
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; |
||||
@ -0,0 +1,16 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
@ -0,0 +1,35 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
DROP TABLE IF EXISTS t_ds_process_task_lineage; |
||||
CREATE TABLE t_ds_process_task_lineage ( |
||||
id int NOT NULL, |
||||
process_definition_code bigint NOT NULL DEFAULT 0, |
||||
process_definition_version int NOT NULL DEFAULT 0, |
||||
task_definition_code bigint NOT NULL DEFAULT 0, |
||||
task_definition_version int NOT NULL DEFAULT 0, |
||||
dept_project_code bigint NOT NULL DEFAULT 0, |
||||
dept_process_definition_code bigint NOT NULL DEFAULT 0, |
||||
dept_task_definition_code bigint NOT NULL DEFAULT 0, |
||||
create_time timestamp NOT NULL DEFAULT current_timestamp, |
||||
update_time timestamp NOT NULL DEFAULT current_timestamp, |
||||
PRIMARY KEY (id) |
||||
); |
||||
|
||||
create index idx_process_code_version on t_ds_process_task_lineage (process_definition_code,process_definition_version); |
||||
create index idx_task_code_version on t_ds_process_task_lineage (task_definition_code,task_definition_version); |
||||
create index idx_dept_code on t_ds_process_task_lineage (dept_project_code,dept_process_definition_code,dept_task_definition_code); |
||||
@ -0,0 +1,16 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
@ -0,0 +1,31 @@
|
||||
#!/bin/bash |
||||
# |
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
# |
||||
|
||||
BIN_DIR=$(dirname $0) |
||||
DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/../..; pwd)} |
||||
|
||||
if [ "$DOCKER" != "true" ]; then |
||||
source "$DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh" |
||||
fi |
||||
|
||||
JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms4g -Xmx4g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} |
||||
|
||||
$JAVA_HOME/bin/java $JAVA_OPTS \ |
||||
-cp "$DOLPHINSCHEDULER_HOME/tools/conf":"$DOLPHINSCHEDULER_HOME/tools/libs/*":"$DOLPHINSCHEDULER_HOME/tools/sql" \ |
||||
-Dspring.profiles.active=lineage,${DATABASE} \ |
||||
org.apache.dolphinscheduler.tools.lineage.MigrateLineage |
||||
@ -0,0 +1,56 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.tools.lineage; |
||||
|
||||
import org.apache.dolphinscheduler.tools.resource.MigrateResource; |
||||
|
||||
import java.sql.SQLException; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.boot.CommandLineRunner; |
||||
import org.springframework.boot.SpringApplication; |
||||
import org.springframework.boot.autoconfigure.SpringBootApplication; |
||||
import org.springframework.context.annotation.ComponentScan; |
||||
import org.springframework.context.annotation.Profile; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@SpringBootApplication |
||||
@ComponentScan("org.apache.dolphinscheduler") |
||||
@Slf4j |
||||
public class MigrateLineage { |
||||
|
||||
public static void main(String[] args) { |
||||
SpringApplication.run(MigrateResource.class, args); |
||||
} |
||||
|
||||
@Component |
||||
@Profile("lineage") |
||||
static class MigrateLineageRunner implements CommandLineRunner { |
||||
|
||||
@Autowired |
||||
private MigrateLineageService migrateLineageService; |
||||
|
||||
@Override |
||||
public void run(String... args) throws SQLException { |
||||
log.info("Starting Migrate lineage data..."); |
||||
migrateLineageService.migrateLineageOnce(); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,120 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.tools.lineage; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskLineage; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; |
||||
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; |
||||
import org.apache.dolphinscheduler.dao.repository.ProcessTaskLineageDao; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; |
||||
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; |
||||
import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory; |
||||
|
||||
import org.apache.commons.collections.CollectionUtils; |
||||
|
||||
import java.sql.SQLException; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
@Service |
||||
@Slf4j |
||||
public class MigrateLineageService { |
||||
|
||||
@Autowired |
||||
private TaskDefinitionMapper taskDefinitionMapper; |
||||
|
||||
@Autowired |
||||
private ProcessTaskLineageDao processTaskLineageDao; |
||||
|
||||
@Autowired |
||||
private ProcessTaskRelationMapper processTaskRelationMapper; |
||||
|
||||
@Autowired |
||||
private ProcessDefinitionMapper processDefinitionMapper; |
||||
|
||||
public void migrateLineageOnce() { |
||||
try { |
||||
List<ProcessTaskLineage> processTaskLineageList = getAllProcessLineages(); |
||||
int insertResult = processTaskLineageDao.batchInsert(processTaskLineageList); |
||||
if (insertResult > 0) { |
||||
log.info("Migrate lineage successfully, insert count: {}", insertResult); |
||||
} else { |
||||
log.info("No lineage to migrate."); |
||||
} |
||||
} catch (Exception e) { |
||||
log.error("Failed to migrate lineage:", e); |
||||
} |
||||
} |
||||
|
||||
private List<ProcessTaskLineage> getAllProcessLineages() throws SQLException { |
||||
List<TaskDefinition> taskDefinitionList = |
||||
taskDefinitionMapper.queryDefinitionsByTaskType(DependentLogicTaskChannelFactory.NAME); |
||||
List<ProcessTaskRelation> processTaskRelationList = |
||||
processTaskRelationMapper.queryByTaskCodes(taskDefinitionList.stream() |
||||
.map(TaskDefinition::getCode).toArray(Long[]::new)); |
||||
List<ProcessTaskLineage> processTaskLineageList = new ArrayList<>(); |
||||
|
||||
for (TaskDefinition taskDefinition : taskDefinitionList) { |
||||
parseDependentTaskParams(taskDefinition, processTaskLineageList); |
||||
|
||||
for (ProcessTaskLineage processTaskLineage : processTaskLineageList) { |
||||
processTaskLineage.setProcessDefinitionCode(processTaskRelationList.stream() |
||||
.filter(processTaskRelation -> processTaskRelation.getPreTaskCode() == taskDefinition.getCode() |
||||
|| processTaskRelation.getPostTaskCode() == taskDefinition.getCode()) |
||||
.findFirst().get().getProcessDefinitionCode()); |
||||
processTaskLineage.setProcessDefinitionVersion(processTaskRelationList.stream() |
||||
.filter(processTaskRelation -> processTaskRelation.getPreTaskCode() == taskDefinition.getCode() |
||||
|| processTaskRelation.getPostTaskCode() == taskDefinition.getCode()) |
||||
.findFirst().get().getProcessDefinitionVersion()); |
||||
} |
||||
} |
||||
return processTaskLineageList; |
||||
} |
||||
|
||||
private void parseDependentTaskParams(TaskDefinition taskDefinition, List<ProcessTaskLineage> taskLineageList) { |
||||
DependentParameters dependentParameters = |
||||
JSONUtils.parseObject(taskDefinition.getTaskParams(), DependentParameters.class); |
||||
if (dependentParameters != null) { |
||||
List<DependentTaskModel> dependTaskList = |
||||
dependentParameters.getDependence().getDependTaskList(); |
||||
if (!CollectionUtils.isEmpty(dependTaskList)) { |
||||
for (DependentTaskModel taskModel : dependTaskList) { |
||||
List<DependentItem> dependItemList = taskModel.getDependItemList(); |
||||
for (DependentItem dependentItem : dependItemList) { |
||||
ProcessTaskLineage processTaskLineage = new ProcessTaskLineage(); |
||||
processTaskLineage.setDeptProjectCode(dependentItem.getProjectCode()); |
||||
processTaskLineage.setDeptProcessDefinitionCode(dependentItem.getDefinitionCode()); |
||||
processTaskLineage.setDeptTaskDefinitionCode(dependentItem.getDepTaskCode()); |
||||
taskLineageList.add(processTaskLineage); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue