Browse Source

test

pull/2/head
qiaozhanwei 5 years ago
parent
commit
41d8abdafb
  1. 296
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java
  2. 192
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java
  3. 107
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java
  4. 24
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  5. 36
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue
  6. 36
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue
  7. 18
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue
  8. 21
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue
  9. 38
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue
  10. 2
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue
  11. 13
      dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js
  12. 2
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
  13. 207
      pom.xml

296
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentExecute.java

@ -0,0 +1,296 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.task.dependent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.DependentRelation;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.DependentItem;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.utils.DependentUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
/**
* dependent item execute
*/
public class DependentExecute {
/**
* process service
*/
private final ProcessService processService = SpringApplicationContext.getBean(ProcessService.class);
/**
* depend item list
*/
private List<DependentItem> dependItemList;
/**
* dependent relation
*/
private DependentRelation relation;
/**
* depend result
*/
private DependResult modelDependResult = DependResult.WAITING;
/**
* depend result map
*/
private Map<String, DependResult> dependResultMap = new HashMap<>();
/**
* logger
*/
private Logger logger = LoggerFactory.getLogger(DependentExecute.class);
/**
* constructor
* @param itemList item list
* @param relation relation
*/
public DependentExecute(List<DependentItem> itemList, DependentRelation relation){
this.dependItemList = itemList;
this.relation = relation;
}
/**
* get dependent item for one dependent item
* @param dependentItem dependent item
* @param currentTime current time
* @return DependResult
*/
private DependResult getDependentResultForItem(DependentItem dependentItem, Date currentTime){
List<DateInterval> dateIntervals = DependentUtils.getDateIntervalList(currentTime, dependentItem.getDateValue());
return calculateResultForTasks(dependentItem, dateIntervals );
}
/**
* calculate dependent result for one dependent item.
* @param dependentItem dependent item
* @param dateIntervals date intervals
* @return dateIntervals
*/
private DependResult calculateResultForTasks(DependentItem dependentItem,
List<DateInterval> dateIntervals) {
DependResult result = DependResult.FAILED;
for(DateInterval dateInterval : dateIntervals){
ProcessInstance processInstance = findLastProcessInterval(dependentItem.getDefinitionId(),
dateInterval);
if(processInstance == null){
logger.error("cannot find the right process instance: definition id:{}, start:{}, end:{}",
dependentItem.getDefinitionId(), dateInterval.getStartTime(), dateInterval.getEndTime() );
return DependResult.FAILED;
}
// need to check workflow for updates, so get all task and check the task state
if(dependentItem.getDepTasks().equals(Constants.DEPENDENT_ALL)){
List<TaskNode> taskNodes =
processService.getTaskNodeListByDefinitionId(dependentItem.getDefinitionId());
if(taskNodes != null && taskNodes.size() > 0){
List<DependResult> results = new ArrayList<>();
DependResult tmpResult = DependResult.FAILED;
for(TaskNode taskNode:taskNodes){
tmpResult = getDependTaskResult(taskNode.getName(),processInstance);
if(DependResult.FAILED == tmpResult){
break;
}else{
results.add(getDependTaskResult(taskNode.getName(),processInstance));
}
}
if(DependResult.FAILED == tmpResult){
result = DependResult.FAILED;
}else if(results.contains(DependResult.WAITING)){
result = DependResult.WAITING;
}else{
result = DependResult.SUCCESS;
}
}else{
result = DependResult.FAILED;
}
}else{
result = getDependTaskResult(dependentItem.getDepTasks(),processInstance);
}
if(result != DependResult.SUCCESS){
break;
}
}
return result;
}
/**
* get depend task result
* @param taskName
* @param processInstance
* @return
*/
private DependResult getDependTaskResult(String taskName, ProcessInstance processInstance) {
DependResult result = DependResult.FAILED;
TaskInstance taskInstance = null;
List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
for(TaskInstance task : taskInstanceList){
if(task.getName().equals(taskName)){
taskInstance = task;
break;
}
}
if(taskInstance == null){
// cannot find task in the process instance
// maybe because process instance is running or failed.
result = getDependResultByProcessStateWhenTaskNull(processInstance.getState());
}else{
result = getDependResultByState(taskInstance.getState());
}
return result;
}
/**
* find the last one process instance that :
* 1. manual run and finish between the interval
* 2. schedule run and schedule time between the interval
* @param definitionId definition id
* @param dateInterval date interval
* @return ProcessInstance
*/
private ProcessInstance findLastProcessInterval(int definitionId, DateInterval dateInterval) {
ProcessInstance runningProcess = processService.findLastRunningProcess(definitionId, dateInterval);
if(runningProcess != null){
return runningProcess;
}
ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval(
definitionId, dateInterval
);
ProcessInstance lastManualProcess = processService.findLastManualProcessInterval(
definitionId, dateInterval
);
if(lastManualProcess ==null){
return lastSchedulerProcess;
}
if(lastSchedulerProcess == null){
return lastManualProcess;
}
return (lastManualProcess.getEndTime().after(lastSchedulerProcess.getEndTime()))?
lastManualProcess : lastSchedulerProcess;
}
/**
* get dependent result by task/process instance state
* @param state state
* @return DependResult
*/
private DependResult getDependResultByState(ExecutionStatus state) {
if(state.typeIsRunning()
|| state == ExecutionStatus.SUBMITTED_SUCCESS
|| state == ExecutionStatus.WAITTING_THREAD){
return DependResult.WAITING;
}else if(state.typeIsSuccess()){
return DependResult.SUCCESS;
}else{
return DependResult.FAILED;
}
}
/**
* get dependent result by task instance state when task instance is null
* @param state state
* @return DependResult
*/
private DependResult getDependResultByProcessStateWhenTaskNull(ExecutionStatus state) {
if(state.typeIsRunning()
|| state == ExecutionStatus.SUBMITTED_SUCCESS
|| state == ExecutionStatus.WAITTING_THREAD){
return DependResult.WAITING;
}else{
return DependResult.FAILED;
}
}
/**
* judge depend item finished
* @param currentTime current time
* @return boolean
*/
public boolean finish(Date currentTime){
if(modelDependResult == DependResult.WAITING){
modelDependResult = getModelDependResult(currentTime);
return false;
}
return true;
}
/**
* get model depend result
* @param currentTime current time
* @return DependResult
*/
public DependResult getModelDependResult(Date currentTime){
List<DependResult> dependResultList = new ArrayList<>();
for(DependentItem dependentItem : dependItemList){
DependResult dependResult = getDependResultForItem(dependentItem, currentTime);
if(dependResult != DependResult.WAITING){
dependResultMap.put(dependentItem.getKey(), dependResult);
}
dependResultList.add(dependResult);
}
modelDependResult = DependentUtils.getDependResultForRelation(
this.relation, dependResultList
);
return modelDependResult;
}
/**
* get dependent item result
* @param item item
* @param currentTime current time
* @return DependResult
*/
private DependResult getDependResultForItem(DependentItem item, Date currentTime){
String key = item.getKey();
if(dependResultMap.containsKey(key)){
return dependResultMap.get(key);
}
return getDependentResultForItem(item, currentTime);
}
public Map<String, DependResult> getDependResultMap(){
return dependResultMap;
}
}

192
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTask.java

@ -0,0 +1,192 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.worker.task.dependent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.DependentTaskModel;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.DependentUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.slf4j.Logger;
import java.util.*;
import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT;
/**
* Dependent Task
*/
public class DependentTask extends AbstractTask {
/**
* dependent task list
*/
private List<DependentExecute> dependentTaskList = new ArrayList<>();
/**
* depend item result map
* save the result to log file
*/
private Map<String, DependResult> dependResultMap = new HashMap<>();
/**
* dependent parameters
*/
private DependentParameters dependentParameters;
/**
* dependent date
*/
private Date dependentDate;
/**
* process service
*/
private ProcessService processService;
/**
* constructor
* @param props props
* @param logger logger
*/
public DependentTask(TaskProps props, Logger logger) {
super(props, logger);
}
@Override
public void init(){
logger.info("dependent task initialize");
this.dependentParameters = JSONUtils.parseObject(this.taskProps.getDependence(),
DependentParameters.class);
if(dependentParameters != null){
for(DependentTaskModel taskModel : dependentParameters.getDependTaskList()){
this.dependentTaskList.add(new DependentExecute(
taskModel.getDependItemList(), taskModel.getRelation()));
}
}
this.processService = SpringApplicationContext.getBean(ProcessService.class);
if(taskProps.getScheduleTime() != null){
this.dependentDate = taskProps.getScheduleTime();
}else{
this.dependentDate = taskProps.getTaskStartTime();
}
}
@Override
public void handle() throws Exception {
// set the name of the current thread
String threadLoggerInfoName = String.format(Constants.TASK_LOG_INFO_FORMAT, taskProps.getTaskAppId());
Thread.currentThread().setName(threadLoggerInfoName);
try{
TaskInstance taskInstance = null;
while(Stopper.isRunning()){
taskInstance = processService.findTaskInstanceById(this.taskProps.getTaskInstId());
if(taskInstance == null){
exitStatusCode = -1;
break;
}
if(taskInstance.getState() == ExecutionStatus.KILL){
this.cancel = true;
}
if(this.cancel || allDependentTaskFinish()){
break;
}
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
}
if(cancel){
exitStatusCode = Constants.EXIT_CODE_KILL;
}else{
DependResult result = getTaskDependResult();
exitStatusCode = (result == DependResult.SUCCESS) ?
Constants.EXIT_CODE_SUCCESS : Constants.EXIT_CODE_FAILURE;
}
}catch (Exception e){
logger.error(e.getMessage(),e);
exitStatusCode = -1;
throw e;
}
}
/**
* get dependent result
* @return DependResult
*/
private DependResult getTaskDependResult(){
List<DependResult> dependResultList = new ArrayList<>();
for(DependentExecute dependentExecute : dependentTaskList){
DependResult dependResult = dependentExecute.getModelDependResult(dependentDate);
dependResultList.add(dependResult);
}
DependResult result = DependentUtils.getDependResultForRelation(
this.dependentParameters.getRelation(), dependResultList
);
return result;
}
/**
* judge all dependent tasks finish
* @return whether all dependent tasks finish
*/
private boolean allDependentTaskFinish(){
boolean finish = true;
for(DependentExecute dependentExecute : dependentTaskList){
for(Map.Entry<String, DependResult> entry: dependentExecute.getDependResultMap().entrySet()) {
if(!dependResultMap.containsKey(entry.getKey())){
dependResultMap.put(entry.getKey(), entry.getValue());
//save depend result to log
logger.info("dependent item complete {} {},{}",
DEPENDENT_SPLIT, entry.getKey(), entry.getValue().toString());
}
}
if(!dependentExecute.finish(dependentDate)){
finish = false;
}
}
return finish;
}
@Override
public void cancelApplication(boolean cancelApplication) throws Exception {
// cancel process
this.cancel = true;
}
@Override
public AbstractParameters getParameters() {
return null;
}
}

107
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/dependent/DependentTaskTest.java

@ -17,47 +17,106 @@
package org.apache.dolphinscheduler.server.worker.task.dependent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.worker.task.TaskProps;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@RunWith(MockitoJUnitRunner.Silent.class)
public class DependentTaskTest {
private static final Logger logger = LoggerFactory.getLogger(DependentTaskTest.class);
private ProcessService processService;
private ApplicationContext applicationContext;
@Before
public void before() throws Exception{
processService = Mockito.mock(ProcessService.class);
Mockito.when(processService
.findLastRunningProcess(4,DependentDateUtils.getTodayInterval(new Date()).get(0)))
.thenReturn(findLastProcessInterval());
Mockito.when(processService
.getTaskNodeListByDefinitionId(4))
.thenReturn(getTaskNodes());
Mockito.when(processService
.findValidTaskListByProcessId(11))
.thenReturn(getTaskInstances());
Mockito.when(processService
.findTaskInstanceById(252612))
.thenReturn(getTaskInstance());
applicationContext = Mockito.mock(ApplicationContext.class);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
}
@Test
public void testDependInit() throws Exception{
public void test() throws Exception{
TaskProps taskProps = new TaskProps();
String dependString = "{\n" +
"\"dependTaskList\":[\n" +
" {\n" +
" \"dependItemList\":[\n" +
" {\n" +
" \"definitionId\": 101,\n" +
" \"depTasks\": \"ALL\",\n" +
" \"cycle\": \"day\",\n" +
" \"dateValue\": \"last1Day\"\n" +
" }\n" +
" ],\n" +
" \"relation\": \"AND\"\n" +
" }\n" +
" ],\n" +
"\"relation\":\"OR\"\n" +
"}";
taskProps.setTaskInstanceId(252612);
String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"dateValue\":\"today\",\"depTasks\":\"ALL\",\"projectId\":1,\"definitionList\":[{\"label\":\"C\",\"value\":4},{\"label\":\"B\",\"value\":3},{\"label\":\"A\",\"value\":2}],\"cycle\":\"day\",\"definitionId\":4}],\"relation\":\"AND\"}],\"relation\":\"AND\"}";
taskProps.setTaskInstId(252612);
taskProps.setDependence(dependString);
// DependentTask dependentTask = new DependentTask(taskProps, logger);
// dependentTask.init();
// dependentTask.handle();
// Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_FAILURE );
taskProps.setTaskStartTime(new Date());
DependentTask dependentTask = new DependentTask(taskProps, logger);
dependentTask.init();
dependentTask.handle();
Assert.assertEquals(dependentTask.getExitStatusCode(), Constants.EXIT_CODE_SUCCESS );
}
private ProcessInstance findLastProcessInterval(){
ProcessInstance processInstance = new ProcessInstance();
processInstance.setId(11);
processInstance.setState(ExecutionStatus.SUCCESS);
return processInstance;
}
private List<TaskNode> getTaskNodes(){
List<TaskNode> list = new ArrayList<>();
TaskNode taskNode = new TaskNode();
taskNode.setName("C");
taskNode.setType("SQL");
list.add(taskNode);
return list;
}
private List<TaskInstance> getTaskInstances(){
List<TaskInstance> list = new ArrayList<>();
TaskInstance taskInstance = new TaskInstance();
taskInstance.setName("C");
taskInstance.setState(ExecutionStatus.SUCCESS);
taskInstance.setDependency("1231");
list.add(taskInstance);
return list;
}
private TaskInstance getTaskInstance(){
TaskInstance taskInstance = new TaskInstance();
taskInstance.setId(252612);
taskInstance.setName("C");
taskInstance.setState(ExecutionStatus.SUCCESS);
return taskInstance;
}
}

24
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -230,6 +230,30 @@ public class ProcessService {
return processInstanceMapper.queryDetailById(processId);
}
/**
* get task node list by definitionId
* @param defineId
* @return
*/
public List<TaskNode> getTaskNodeListByDefinitionId(Integer defineId){
ProcessDefinition processDefinition = processDefineMapper.selectById(defineId);
if (processDefinition == null) {
logger.info("process define not exists");
return null;
}
String processDefinitionJson = processDefinition.getProcessDefinitionJson();
ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class);
//process data check
if (null == processData) {
logger.error("process data is null");
return null;
}
return processData.getTasks();
}
/**
* find process instance by id
* @param processId processId

36
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue

@ -218,6 +218,19 @@
},
mixins: [disabledState],
methods: {
/**
* getResourceId
*/
marjarId(name) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+name
}).then(res => {
this.mainJar = res.id
}).catch(e => {
this.$message.error(e.msg || '')
})
},
/**
* return localParams
*/
@ -366,7 +379,13 @@
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.mainClass = o.params.mainClass || ''
this.mainJar = o.params.mainJar && o.params.mainJar.id ? o.params.mainJar.id : ''
if(o.params.mainJar.res) {
this.marjarId(o.params.mainJar.res)
} else if(o.params.mainJar.res=='') {
this.mainJar = ''
} else {
this.mainJar = o.params.mainJar.id || ''
}
this.deployMode = o.params.deployMode || ''
this.slot = o.params.slot || 1
this.taskManager = o.params.taskManager || '2'
@ -380,8 +399,19 @@
// backfill resourceList
let resourceList = o.params.resourceList || []
if (resourceList.length) {
this.resourceList = _.map(resourceList, v => {
return v.id
_.map(resourceList, v => {
if(v.res) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+v.res
}).then(res => {
this.resourceList.push(res.id)
}).catch(e => {
this.$message.error(e.msg || '')
})
} else {
this.resourceList.push(v.id)
}
})
this.cacheResourceList = resourceList
}

36
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue

@ -142,6 +142,19 @@
},
mixins: [disabledState],
methods: {
/**
* getResourceId
*/
marjarId(name) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+name
}).then(res => {
this.mainJar = res.id
}).catch(e => {
this.$message.error(e.msg || '')
})
},
/**
* return localParams
*/
@ -245,7 +258,13 @@
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.mainClass = o.params.mainClass || ''
this.mainJar = o.params.mainJar.id || ''
if(o.params.mainJar.res) {
this.marjarId(o.params.mainJar.res)
} else if(o.params.mainJar.res=='') {
this.mainJar = ''
} else {
this.mainJar = o.params.mainJar.id || ''
}
this.mainArgs = o.params.mainArgs || ''
this.others = o.params.others
this.programType = o.params.programType || 'JAVA'
@ -253,8 +272,19 @@
// backfill resourceList
let resourceList = o.params.resourceList || []
if (resourceList.length) {
this.resourceList = _.map(resourceList, v => {
return v.id
_.map(resourceList, v => {
if(v.res) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+v.res
}).then(res => {
this.resourceList.push(res.id)
}).catch(e => {
this.$message.error(e.msg || '')
})
} else {
this.resourceList.push(v.id)
}
})
this.cacheResourceList = resourceList
}

18
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue

@ -161,8 +161,7 @@
resourceList: _.map(this.resourceList, v => {
return {id: v}
}),
localParams: this.localParams,
rawScript: editor ? editor.getValue() : ''
localParams: this.localParams
}
}
},
@ -176,8 +175,19 @@
// backfill resourceList
let resourceList = o.params.resourceList || []
if (resourceList.length) {
this.resourceList = _.map(resourceList, v => {
return v.id
_.map(resourceList, v => {
if(v.res) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+v.res
}).then(res => {
this.resourceList.push(res.id)
}).catch(e => {
this.$message.error(e.msg || '')
})
} else {
this.resourceList.push(v.id)
}
})
this.cacheResourceList = resourceList
}

21
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@ -95,7 +95,7 @@
return {
label: node.name
}
},
}
}
},
mixins: [disabledState],
@ -221,8 +221,7 @@
resourceList: _.map(this.resourceList, v => {
return {id: v}
}),
localParams: this.localParams,
rawScript: editor ? editor.getValue() : ''
localParams: this.localParams
}
}
},
@ -238,12 +237,22 @@
// backfill resourceList
let resourceList = o.params.resourceList || []
if (resourceList.length) {
this.resourceList = _.map(resourceList, v => {
return v.id
_.map(resourceList, v => {
if(v.res) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+v.res
}).then(res => {
this.resourceList.push(res.id)
}).catch(e => {
this.$message.error(e.msg || '')
})
} else {
this.resourceList.push(v.id)
}
})
this.cacheResourceList = resourceList
}
// backfill localParams
let localParams = o.params.localParams || []
if (localParams.length) {

38
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue

@ -261,6 +261,19 @@
},
mixins: [disabledState],
methods: {
/**
* getResourceId
*/
marjarId(name) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+name
}).then(res => {
this.mainJar = res.id
}).catch(e => {
this.$message.error(e.msg || '')
})
},
/**
* return localParams
*/
@ -414,7 +427,13 @@
// Non-null objects represent backfill
if (!_.isEmpty(o)) {
this.mainClass = o.params.mainClass || ''
this.mainJar = o.params.mainJar && o.params.mainJar.id ? o.params.mainJar.id : ''
if(o.params.mainJar.res) {
this.marjarId(o.params.mainJar.res)
} else if(o.params.mainJar.res=='') {
this.mainJar = ''
} else {
this.mainJar = o.params.mainJar.id || ''
}
this.deployMode = o.params.deployMode || ''
this.driverCores = o.params.driverCores || 1
this.driverMemory = o.params.driverMemory || '512M'
@ -429,9 +448,20 @@
// backfill resourceList
let resourceList = o.params.resourceList || []
if (resourceList.length) {
this.resourceList = _.map(resourceList, v => {
return v.id
})
_.map(resourceList, v => {
if(v.res) {
this.store.dispatch('dag/getResourceId',{
type: 'FILE',
fullName: '/'+v.res
}).then(res => {
this.resourceList.push(res.id)
}).catch(e => {
this.$message.error(e.msg || '')
})
} else {
this.resourceList.push(v.id)
}
})
this.cacheResourceList = resourceList
}

2
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/list/_source/list.vue

@ -22,7 +22,7 @@
<th scope="col" width="50">
<x-checkbox @on-change="_topCheckBoxClick" v-model="checkAll"></x-checkbox>
</th>
<th scope="col">
<th scope="col" width="30">
<span>{{$t('#')}}</span>
</th>
<th scope="col" width="70">

13
dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js

@ -715,5 +715,14 @@ export default {
reject(e)
})
})
}
}
},
getResourceId ({ state }, payload) {
return new Promise((resolve, reject) => {
io.get(`resources/queryResource`, payload, res => {
resolve(res.data)
}).catch(e => {
reject(e)
})
})
},
}

2
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@ -115,7 +115,7 @@ export default {
'SQL Type': 'sql类型',
'Title': '主题',
'Please enter the title of email': '请输入邮件主题',
'Table': '',
'Table': '',
'Attachment': '附件',
'SQL Parameter': 'sql参数',
'SQL Statement': 'sql语句',

207
pom.xml

@ -683,183 +683,70 @@
<version>${maven-surefire-plugin.version}</version>
<configuration>
<includes>
<include>**/alert/template/AlertTemplateFactoryTest.java</include>
<include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
<include>**/alert/utils/EnterpriseWeChatUtilsTest.java</include>
<include>**/alert/utils/ExcelUtilsTest.java</include>
<include>**/alert/utils/FuncUtilsTest.java</include>
<include>**/alert/utils/JSONUtilsTest.java</include>
<include>**/alert/utils/MailUtilsTest.java</include>
<include>**/alert/utils/PropertyUtilsTest.java</include>
<!--<include>**/api/controller/AccessTokenControllerTest.java</include>-->
<!--<include>**/api/controller/AlertGroupControllerTest.java</include>-->
<!--<include>**/api/controller/DataAnalysisControllerTest.java</include>-->
<!--<include>**/api/controller/DataSourceControllerTest.java</include>-->
<!--<include>**/api/controller/ExecutorControllerTest.java</include>-->
<!--<include>**/api/controller/LoggerControllerTest.java</include>-->
<!--<include>**/api/controller/LoginControllerTest.java</include>-->
<!--<include>**/api/controller/MonitorControllerTest.java</include>-->
<!--<include>**/api/controller/ProcessDefinitionControllerTest.java</include>-->
<!--<include>**/api/controller/ProcessInstanceControllerTest.java</include>-->
<!--<include>**/api/controller/ProjectControllerTest.java</include>-->
<!--<include>**/api/controller/QueueControllerTest.java</include>-->
<!--<include>**/api/controller/ResourcesControllerTest.java</include>-->
<!--<include>**/api/controller/SchedulerControllerTest.java</include>-->
<!--<include>**/api/controller/TaskInstanceControllerTest.java</include>-->
<!--<include>**/api/controller/TaskRecordControllerTest.java</include>-->
<!--<include>**/api/controller/TenantControllerTest.java</include>-->
<!--<include>**/api/controller/UsersControllerTest.java</include>-->
<!--<include>**/api/controller/WorkerGroupControllerTest.java</include>-->
<include>**/api/dto/resources/filter/ResourceFilterTest.java</include>
<include>**/api/dto/resources/visitor/ResourceTreeVisitorTest.java</include>
<include>**/api/enums/testGetEnum.java</include>
<include>**/api/enums/StatusTest.java</include>
<include>**/api/interceptor/LoginHandlerInterceptorTest.java</include>
<include>**/api/security/PasswordAuthenticatorTest.java</include>
<include>**/api/security/SecurityConfigTest.java</include>
<include>**/common/utils/*.java</include>
<include>**/common/utils/process/ProcessBuilderForWin32Test.java</include>
<include>**/common/utils/process/ProcessEnvironmentForWin32Test.java</include>
<include>**/common/utils/process/ProcessImplForWin32Test.java</include>
<include>**/common/log/*.java</include>
<include>**/common/threadutils/*.java</include>
<include>**/common/graph/*.java</include>
<include>**/common/queue/*.java</include>
<include>**/common/task/FlinkParametersTest.java</include>
<include>**/common/task/SqoopParameterEntityTest.java</include>
<include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/utils/FileUtilsTest.java</include>
<include>**/api/utils/FourLetterWordTest.java</include>
<include>**/api/utils/exportprocess/DataSourceParamTest.java</include>
<include>**/api/utils/exportprocess/DependentParamTest.java</include>
<include>**/api/enums/*.java</include>
<include>**/api/controller/DataSourceControllerTest.java</include>
<include>**/api/service/AccessTokenServiceTest.java</include>
<include>**/api/service/QueueServiceTest.java</include>
<include>**/api/service/MonitorServiceTest.java</include>
<include>**/api/service/SessionServiceTest.java</include>
<include>**/api/service/UsersServiceTest.java</include>
<include>**/api/service/TenantServiceTest.java</include>
<include>**/api/service/WorkerGroupServiceTest.java</include>
<include>**/api/service/AlertGroupServiceTest.java</include>
<include>**/api/service/BaseDAGServiceTest.java</include>
<include>**/api/service/BaseServiceTest.java</include>
<include>**/api/service/DataAnalysisServiceTest.java</include>
<include>**/api/service/DataSourceServiceTest.java</include>
<include>**/api/service/UserAlertGroupServiceTest.java</include>
<include>**/api/service/ProjectServiceTest.java</include>
<include>**/api/service/ProcessDefinitionServiceTest.java</include>
<include>**/api/service/UdfFuncServiceTest.java</include>
<include>**/api/service/ResourcesServiceTest.java</include>
<include>**/api/service/ExecutorService2Test.java</include>
<include>**/api/service/ExecutorServiceTest.java</include>
<include>**/api/service/BaseServiceTest.java</include>
<include>**/api/service/BaseDAGServiceTest.java</include>
<include>**/api/service/LoggerServiceTest.java</include>
<include>**/api/service/MonitorServiceTest.java</include>
<include>**/api/service/ProcessDefinitionServiceTest.java</include>
<include>**/api/service/DataAnalysisServiceTest.java</include>
<include>**/api/service/ProcessInstanceServiceTest.java</include>
<include>**/api/service/ProjectServiceTest.java</include>
<include>**/api/service/QueueServiceTest.java</include>
<include>**/api/service/ResourcesServiceTest.java</include>
<include>**/api/service/SchedulerServiceTest.java</include>
<include>**/api/service/SessionServiceTest.java</include>
<include>**/api/service/TaskInstanceServiceTest.java</include>
<include>**/api/service/TenantServiceTest.java</include>
<include>**/api/service/UdfFuncServiceTest.java</include>
<include>**/api/service/UserAlertGroupServiceTest.java</include>
<include>**/api/service/UsersServiceTest.java</include>
<include>**/api/service/WorkerGroupServiceTest.java</include>
<include>**/api/utils/exportprocess/DataSourceParamTest.java</include>
<include>**/api/utils/exportprocess/DependentParamTest.java</include>
<include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/utils/FileUtilsTest.java</include>
<include>**/api/utils/FourLetterWordMainTest.java</include>
<include>**/api/utils/ZookeeperMonitorUtilsTest.java</include>
<include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/HttpClientTest.java</include>
<include>**/common/graph/DAGTest.java</include>
<include>**/common/os/OshiTest.java</include>
<include>**/common/os/OSUtilsTest.java</include>
<include>**/common/shell/ShellExecutorTest.java</include>
<include>**/common/task/EntityTestUtils.java</include>
<include>**/common/task/FlinkParametersTest.java</include>
<include>**/common/task/SqoopParameterEntityTest.java</include>
<include>**/common/threadutils/ThreadPoolExecutorsTest.java</include>
<include>**/common/threadutils/ThreadUtilsTest.java</include>
<include>**/common/utils/placeholder/TimePlaceholderUtilsTest.java</include>
<include>**/common/utils/process/ProcessBuilderForWin32Test.java</include>
<include>**/common/utils/process/ProcessEnvironmentForWin32Test.java</include>
<include>**/common/utils/process/ProcessImplForWin32Test.java</include>
<include>**/common/utils/CollectionUtilsTest.java</include>
<include>**/common/utils/CommonUtilsTest.java</include>
<include>**/common/utils/DateUtilsTest.java</include>
<include>**/common/utils/DependentUtilsTest.java</include>
<include>**/common/utils/EncryptionUtilsTest.java</include>
<include>**/common/utils/FileUtilsTest.java</include>
<include>**/common/utils/HadoopUtilsTest.java</include>
<include>**/common/utils/IpUtilsTest.java</include>
<include>**/common/utils/JSONUtilsTest.java</include>
<include>**/common/utils/LoggerUtilsTest.java</include>
<include>**/common/utils/OSUtilsTest.java</include>
<include>**/common/utils/ParameterUtilsTest.java</include>
<include>**/common/utils/PreconditionsTest.java</include>
<include>**/common/utils/PropertyUtilsTest.java</include>
<include>**/common/utils/SchemaUtilsTest.java</include>
<include>**/common/utils/ScriptRunnerTest.java</include>
<include>**/common/utils/SensitiveLogUtilsTest.java</include>
<include>**/common/utils/StringTest.java</include>
<include>**/common/utils/StringUtilsTest.java</include>
<include>**/common/utils/TaskParametersUtilsTest.java</include>
<include>**/common/ConstantsTest.java</include>
<include>**/alert/utils/ExcelUtilsTest.java</include>
<include>**/alert/utils/FuncUtilsTest.java</include>
<include>**/alert/utils/JSONUtilsTest.java</include>
<include>**/alert/utils/PropertyUtilsTest.java</include>
<include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/server/utils/ParamUtilsTest.java</include>
<include>**/server/master/MasterExecThreadTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
<include>**/dao/mapper/AlertGroupMapperTest.java</include>
<include>**/dao/mapper/AlertMapperTest.java</include>
<include>**/dao/mapper/CommandMapperTest.java</include>
<include>**/dao/mapper/ConnectionFactoryTest.java</include>
<include>**/dao/mapper/DataSourceMapperTest.java</include>
<include>**/dao/mapper/DataSourceUserMapperTest.java</include>
<include>**/dao/mapper/ErrorCommandMapperTest.java</include>
<include>**/dao/mapper/ProcessDefinitionMapperTest.java</include>
<include>**/dao/mapper/ProcessInstanceMapMapperTest.java</include>
<include>**/dao/mapper/ProcessInstanceMapperTest.java</include>
<include>**/dao/mapper/ProjectMapperTest.java</include>
<include>**/dao/mapper/ProjectUserMapperTest.java</include>
<include>**/dao/mapper/QueueMapperTest.java</include>
<include>**/dao/mapper/ResourceMapperTest.java</include>
<include>**/dao/mapper/ResourceUserMapperTest.java</include>
<include>**/dao/mapper/ScheduleMapperTest.java</include>
<include>**/dao/mapper/SessionMapperTest.java</include>
<include>**/dao/mapper/TaskInstanceMapperTest.java</include>
<include>**/dao/mapper/TenantMapperTest.java</include>
<include>**/dao/mapper/UdfFuncMapperTest.java</include>
<include>**/dao/mapper/UDFUserMapperTest.java</include>
<include>**/dao/mapper/UserAlertGroupMapperTest.java</include>
<include>**/dao/mapper/UserMapperTest.java</include>
<include>**/dao/mapper/WorkerGroupMapperTest.java</include>
<include>**/dao/entity/TaskInstanceTest.java</include>
<include>**/dao/cron/CronUtilsTest.java</include>
<include>**/dao/utils/DagHelperTest.java</include>
<include>**/dao/AlertDaoTest.java</include>
<include>**/remote/FastJsonSerializerTest.java</include>
<include>**/remote/NettyRemotingClientTest.java</include>
<include>**/remote/ResponseFutureTest.java</include>
<include>**/server/log/MasterLogFilterTest.java</include>
<include>**/server/log/SensitiveDataConverterTest.java</include>
<include>**/server/log/TaskLogDiscriminatorTest.java</include>
<include>**/server/log/TaskLogFilterTest.java</include>
<include>**/server/log/WorkerLogFilterTest.java</include>
<include>**/server/master/dispatch/executor/NettyExecutorManagerTest.java</include>
<include>**/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java</include>
<include>**/server/master/dispatch/host/assign/RandomSelectorTest.java</include>
<include>**/server/master/dispatch/host/assign/RoundRobinSelectorTest.java</include>
<include>**/server/master/dispatch/host/RoundRobinHostManagerTest.java</include>
<include>**/server/master/dispatch/ExecutorDispatcherTest.java</include>
<include>**/server/master/register/MasterRegistryTest.java</include>
<include>**/server/master/AlertManagerTest.java</include>
<include>**/server/master/MasterCommandTest.java</include>
<include>**/server/master/MasterExecThreadTest.java</include>
<include>**/server/master/ParamsTest.java</include>
<include>**/server/register/ZookeeperNodeManagerTest.java</include>
<include>**/server/utils/DataxUtilsTest.java</include>
<include>**/server/utils/ExecutionContextTestUtils.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/server/utils/ParamUtilsTest.java</include>
<include>**/server/utils/ProcessUtilsTest.java</include>
<include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/worker/processor/TaskCallbackServiceTest.java</include>
<include>**/server/worker/registry/WorkerRegistryTest.java</include>
<include>**/server/worker/shell/ShellCommandExecutorTest.java</include>
<include>**/server/worker/sql/SqlExecutorTest.java</include>
<include>**/server/worker/task/datax/DataxTaskTest.java</include>
<include>**/server/worker/task/dependent/DependentTaskTest.java</include>
<include>**/server/worker/task/spark/SparkTaskTest.java</include>
<include>**/server/worker/task/EnvFileTest.java</include>
<include>**/alert/template/AlertTemplateFactoryTest.java</include>
<include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
<include>**/server/worker/task/datax/DataxTaskTest.java</include>
<include>**/server/worker/task/dependent/DependentTaskTest.java</include>
<include>**/server/worker/task/shell/ShellTaskTest.java</include>
<include>**/server/worker/task/spark/SparkTaskTest.java</include>
<include>**/server/worker/task/sqoop/SqoopTaskTest.java</include>
<include>**/server/worker/EnvFileTest.java</include>
<include>**/service/quartz/cron/CronUtilsTest.java</include>
<include>**/server/utils/DataxUtilsTest.java</include>
<include>**/service/zk/DefaultEnsembleProviderTest.java</include>
<include>**/service/zk/ZKServerTest.java</include>
<include>**/service/queue/TaskUpdateQueueTest.java</include>
<include>**/dao/datasource/BaseDataSourceTest.java</include>
<include>**/alert/utils/MailUtilsTest.java</include>
<include>**/dao/AlertDaoTest.java</include>
</includes>
<!-- <skip>true</skip> -->
<argLine>-Xmx2048m</argLine>
<threadCount>3</threadCount>
<forkCount>3</forkCount>
<reuseForks>true</reuseForks>
</configuration>
</plugin>

Loading…
Cancel
Save