Browse Source
* Supports task instance cache operation * add task plugin cache * use SHA-256 to generate key * Update dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql Co-authored-by: Jay Chung <zhongjiajie955@gmail.com> * Update dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql Co-authored-by: Jay Chung <zhongjiajie955@gmail.com> * Optimizing database Scripts * Optimize clear cache operation Co-authored-by: Jay Chung <zhongjiajie955@gmail.com>3.2.0-release
JieguangZhou
2 years ago
committed by
GitHub
77 changed files with 1151 additions and 24 deletions
@ -0,0 +1,44 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.api.dto.taskInstance; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.api.utils.Result; |
||||||
|
|
||||||
|
import lombok.Data; |
||||||
|
|
||||||
|
/** |
||||||
|
* task instance success response |
||||||
|
*/ |
||||||
|
@Data |
||||||
|
public class TaskInstanceRemoveCacheResponse extends Result { |
||||||
|
|
||||||
|
private String cacheKey; |
||||||
|
|
||||||
|
public TaskInstanceRemoveCacheResponse(Result result) { |
||||||
|
super(); |
||||||
|
this.setCode(result.getCode()); |
||||||
|
this.setMsg(result.getMsg()); |
||||||
|
} |
||||||
|
|
||||||
|
public TaskInstanceRemoveCacheResponse(Result result, String cacheKey) { |
||||||
|
super(); |
||||||
|
this.setCode(result.getCode()); |
||||||
|
this.setMsg(result.getMsg()); |
||||||
|
this.cacheKey = cacheKey; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,161 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.dao.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||||
|
|
||||||
|
import org.apache.commons.codec.digest.DigestUtils; |
||||||
|
import org.apache.commons.lang3.StringUtils; |
||||||
|
import org.apache.commons.lang3.tuple.Pair; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Comparator; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
import java.util.Set; |
||||||
|
import java.util.regex.Matcher; |
||||||
|
import java.util.regex.Pattern; |
||||||
|
import java.util.stream.Collectors; |
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode; |
||||||
|
|
||||||
|
public class TaskCacheUtils { |
||||||
|
|
||||||
|
private TaskCacheUtils() { |
||||||
|
throw new IllegalStateException("Utility class"); |
||||||
|
} |
||||||
|
|
||||||
|
public static final String MERGE_TAG = "-"; |
||||||
|
|
||||||
|
/** |
||||||
|
* generate cache key for task instance |
||||||
|
* the follow message will be used to generate cache key |
||||||
|
* 2. task version |
||||||
|
* 3. task is cache |
||||||
|
* 4. input VarPool, from upstream task and workflow global parameters |
||||||
|
* @param taskInstance task instance |
||||||
|
* @param taskExecutionContext taskExecutionContext |
||||||
|
* @return cache key |
||||||
|
*/ |
||||||
|
public static String generateCacheKey(TaskInstance taskInstance, TaskExecutionContext taskExecutionContext) { |
||||||
|
List<String> keyElements = new ArrayList<>(); |
||||||
|
keyElements.add(String.valueOf(taskInstance.getTaskCode())); |
||||||
|
keyElements.add(String.valueOf(taskInstance.getTaskDefinitionVersion())); |
||||||
|
keyElements.add(String.valueOf(taskInstance.getIsCache().getCode())); |
||||||
|
keyElements.add(String.valueOf(taskInstance.getEnvironmentConfig())); |
||||||
|
keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext)); |
||||||
|
String data = StringUtils.join(keyElements, "_"); |
||||||
|
return DigestUtils.sha256Hex(data); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* generate cache key for task instance which is cache execute |
||||||
|
* this key will record which cache task instance will be copied, and cache key will be used |
||||||
|
* tagCacheKey = sourceTaskId + "-" + cacheKey |
||||||
|
* @param sourceTaskId source task id |
||||||
|
* @param cacheKey cache key |
||||||
|
* @return tagCacheKey |
||||||
|
*/ |
||||||
|
public static String generateTagCacheKey(Integer sourceTaskId, String cacheKey) { |
||||||
|
return sourceTaskId + MERGE_TAG + cacheKey; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* revert cache key tag to source task id and cache key |
||||||
|
* @param tagCacheKey cache key |
||||||
|
* @return Pair<Integer, String>, first is source task id, second is cache key |
||||||
|
*/ |
||||||
|
public static Pair<Integer, String> revertCacheKey(String tagCacheKey) { |
||||||
|
Pair<Integer, String> taskIdAndCacheKey; |
||||||
|
if (tagCacheKey == null) { |
||||||
|
taskIdAndCacheKey = Pair.of(-1, ""); |
||||||
|
return taskIdAndCacheKey; |
||||||
|
} |
||||||
|
if (tagCacheKey.contains(MERGE_TAG)) { |
||||||
|
String[] split = tagCacheKey.split(MERGE_TAG); |
||||||
|
if (split.length == 2) { |
||||||
|
taskIdAndCacheKey = Pair.of(Integer.parseInt(split[0]), split[1]); |
||||||
|
} else { |
||||||
|
taskIdAndCacheKey = Pair.of(-1, ""); |
||||||
|
} |
||||||
|
return taskIdAndCacheKey; |
||||||
|
} else { |
||||||
|
return Pair.of(-1, tagCacheKey); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get hash data of task input var pool |
||||||
|
* there are two parts of task input var pool: from upstream task and workflow global parameters |
||||||
|
* @param taskInstance task instance |
||||||
|
* taskExecutionContext taskExecutionContext |
||||||
|
*/ |
||||||
|
public static String getTaskInputVarPoolData(TaskInstance taskInstance, TaskExecutionContext context) { |
||||||
|
JsonNode taskParams = JSONUtils.parseObject(taskInstance.getTaskParams()); |
||||||
|
|
||||||
|
// The set of input values considered from localParams in the taskParams
|
||||||
|
Set<String> propertyInSet = JSONUtils.toList(taskParams.get("localParams").toString(), Property.class).stream() |
||||||
|
.filter(property -> property.getDirect().equals(Direct.IN)) |
||||||
|
.map(Property::getProp).collect(Collectors.toSet()); |
||||||
|
|
||||||
|
// The set of input values considered from `${var}` form task definition
|
||||||
|
propertyInSet.addAll(getScriptVarInSet(taskInstance)); |
||||||
|
|
||||||
|
// var pool value from upstream task
|
||||||
|
List<Property> varPool = JSONUtils.toList(taskInstance.getVarPool(), Property.class); |
||||||
|
|
||||||
|
// var pool value from workflow global parameters
|
||||||
|
if (context.getPrepareParamsMap() != null) { |
||||||
|
Set<String> taskVarPoolSet = varPool.stream().map(Property::getProp).collect(Collectors.toSet()); |
||||||
|
List<Property> globalContextVarPool = context.getPrepareParamsMap().entrySet().stream() |
||||||
|
.filter(entry -> !taskVarPoolSet.contains(entry.getKey())) |
||||||
|
.map(Map.Entry::getValue) |
||||||
|
.collect(Collectors.toList()); |
||||||
|
varPool.addAll(globalContextVarPool); |
||||||
|
} |
||||||
|
|
||||||
|
// only consider var pool value which is in propertyInSet
|
||||||
|
varPool = varPool.stream() |
||||||
|
.filter(property -> property.getDirect().equals(Direct.IN)) |
||||||
|
.filter(property -> propertyInSet.contains(property.getProp())) |
||||||
|
.sorted(Comparator.comparing(Property::getProp)) |
||||||
|
.collect(Collectors.toList()); |
||||||
|
return JSONUtils.toJsonString(varPool); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* get var in set from task definition |
||||||
|
* @param taskInstance task instance |
||||||
|
* @return var in set |
||||||
|
*/ |
||||||
|
public static List<String> getScriptVarInSet(TaskInstance taskInstance) { |
||||||
|
Pattern pattern = Pattern.compile("\\$\\{(.+?)\\}"); |
||||||
|
Matcher matcher = pattern.matcher(taskInstance.getTaskParams()); |
||||||
|
|
||||||
|
List<String> varInSet = new ArrayList<>(); |
||||||
|
while (matcher.find()) { |
||||||
|
varInSet.add(matcher.group(1)); |
||||||
|
} |
||||||
|
return varInSet; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,172 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.dao.utils; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.Flag; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||||
|
|
||||||
|
import org.apache.commons.lang3.tuple.Pair; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Map; |
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions; |
||||||
|
import org.junit.jupiter.api.BeforeEach; |
||||||
|
import org.junit.jupiter.api.Test; |
||||||
|
|
||||||
|
class TaskCacheUtilsTest { |
||||||
|
|
||||||
|
private TaskInstance taskInstance; |
||||||
|
|
||||||
|
private TaskExecutionContext taskExecutionContext; |
||||||
|
|
||||||
|
@BeforeEach |
||||||
|
void setUp() { |
||||||
|
String taskParams = "{\n" + |
||||||
|
" \"localParams\": [\n" + |
||||||
|
" {\n" + |
||||||
|
" \"prop\": \"a\",\n" + |
||||||
|
" \"direct\": \"IN\",\n" + |
||||||
|
" \"type\": \"VARCHAR\",\n" + |
||||||
|
" \"value\": \"\"\n" + |
||||||
|
" },\n" + |
||||||
|
" {\n" + |
||||||
|
" \"prop\": \"b\",\n" + |
||||||
|
" \"direct\": \"IN\",\n" + |
||||||
|
" \"type\": \"VARCHAR\",\n" + |
||||||
|
" \"value\": \"bb\"\n" + |
||||||
|
" }\n" + |
||||||
|
" ],\n" + |
||||||
|
" \"rawScript\": \"echo ${c}\\necho ${d}\",\n" + |
||||||
|
" \"resourceList\": []\n" + |
||||||
|
"}"; |
||||||
|
|
||||||
|
String varPool = "[\n" + |
||||||
|
" {\n" + |
||||||
|
" \"prop\": \"c\",\n" + |
||||||
|
" \"direct\": \"IN\",\n" + |
||||||
|
" \"type\": \"VARCHAR\",\n" + |
||||||
|
" \"value\": \"cc\"\n" + |
||||||
|
" },\n" + |
||||||
|
" {\n" + |
||||||
|
" \"prop\": \"k\",\n" + |
||||||
|
" \"direct\": \"IN\",\n" + |
||||||
|
" \"type\": \"VARCHAR\",\n" + |
||||||
|
" \"value\": \"kk\"\n" + |
||||||
|
" }\n" + |
||||||
|
"]"; |
||||||
|
|
||||||
|
taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setTaskParams(taskParams); |
||||||
|
taskInstance.setVarPool(varPool); |
||||||
|
taskInstance.setTaskCode(123L); |
||||||
|
taskInstance.setTaskDefinitionVersion(1); |
||||||
|
taskInstance.setIsCache(Flag.YES); |
||||||
|
|
||||||
|
taskExecutionContext = new TaskExecutionContext(); |
||||||
|
Property property = new Property(); |
||||||
|
property.setProp("a"); |
||||||
|
property.setDirect(Direct.IN); |
||||||
|
property.setType(DataType.VARCHAR); |
||||||
|
property.setValue("aa"); |
||||||
|
Map<String, Property> prepareParamsMap = new HashMap<>(); |
||||||
|
prepareParamsMap.put("a", property); |
||||||
|
taskExecutionContext.setPrepareParamsMap(prepareParamsMap); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
void testRevertCacheKey() { |
||||||
|
Pair<Integer, String> taskIdAndCacheKey1 = TaskCacheUtils.revertCacheKey(null); |
||||||
|
Assertions.assertEquals(Pair.of(-1, ""), taskIdAndCacheKey1); |
||||||
|
|
||||||
|
Pair<Integer, String> taskIdAndCacheKey2 = TaskCacheUtils.revertCacheKey("123"); |
||||||
|
Assertions.assertEquals(Pair.of(-1, "123"), taskIdAndCacheKey2); |
||||||
|
|
||||||
|
Pair<Integer, String> taskIdAndCacheKey3 = TaskCacheUtils.revertCacheKey("1-123"); |
||||||
|
Assertions.assertEquals(Pair.of(1, "123"), taskIdAndCacheKey3); |
||||||
|
|
||||||
|
Pair<Integer, String> taskIdAndCacheKey4 = TaskCacheUtils.revertCacheKey("1-123-4"); |
||||||
|
Assertions.assertEquals(Pair.of(-1, ""), taskIdAndCacheKey4); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
void testGetScriptVarInSet() { |
||||||
|
List<String> scriptVarInSet = TaskCacheUtils.getScriptVarInSet(taskInstance); |
||||||
|
List<String> except = new ArrayList<>(Arrays.asList("c", "d")); |
||||||
|
Assertions.assertEquals(except, scriptVarInSet); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
void TestGetTaskInputVarPoolData() { |
||||||
|
TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext); |
||||||
|
// only a=aa and c=cc will influence the result,
|
||||||
|
// b=bb is a fixed value, will be considered in task version
|
||||||
|
// k=kk is not in task params, will be ignored
|
||||||
|
String except = |
||||||
|
"[{\"prop\":\"a\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"aa\"},{\"prop\":\"c\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"cc\"}]"; |
||||||
|
Assertions.assertEquals(except, TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext)); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
void TestGenerateCacheKey() { |
||||||
|
String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||||
|
Property propertyI = new Property(); |
||||||
|
propertyI.setProp("i"); |
||||||
|
propertyI.setDirect(Direct.IN); |
||||||
|
propertyI.setType(DataType.VARCHAR); |
||||||
|
propertyI.setValue("ii"); |
||||||
|
taskExecutionContext.getPrepareParamsMap().put("i", propertyI); |
||||||
|
String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||||
|
// i will not influence the result, because task instance not use it
|
||||||
|
Assertions.assertEquals(cacheKeyBase, cacheKeyNew); |
||||||
|
|
||||||
|
Property propertyD = new Property(); |
||||||
|
propertyD.setProp("d"); |
||||||
|
propertyD.setDirect(Direct.IN); |
||||||
|
propertyD.setType(DataType.VARCHAR); |
||||||
|
propertyD.setValue("dd"); |
||||||
|
taskExecutionContext.getPrepareParamsMap().put("i", propertyD); |
||||||
|
String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||||
|
// d will influence the result, because task instance use it
|
||||||
|
Assertions.assertNotEquals(cacheKeyBase, cacheKeyD); |
||||||
|
|
||||||
|
taskInstance.setTaskDefinitionVersion(100); |
||||||
|
String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||||
|
// task definition version is changed, so cache key changed
|
||||||
|
Assertions.assertNotEquals(cacheKeyD, cacheKeyE); |
||||||
|
|
||||||
|
taskInstance.setEnvironmentConfig("export PYTHON_HOME=/bin/python3"); |
||||||
|
String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||||
|
// EnvironmentConfig is changed, so cache key changed
|
||||||
|
Assertions.assertNotEquals(cacheKeyE, cacheKeyF); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
void testGetCacheKey() { |
||||||
|
String cacheKey = TaskCacheUtils.generateTagCacheKey(1, "123"); |
||||||
|
Assertions.assertEquals("1-123", cacheKey); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,109 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.event; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.Flag; |
||||||
|
import org.apache.dolphinscheduler.common.enums.StateEventType; |
||||||
|
import org.apache.dolphinscheduler.common.enums.TaskEventType; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; |
||||||
|
import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils; |
||||||
|
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; |
||||||
|
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; |
||||||
|
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; |
||||||
|
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||||
|
import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; |
||||||
|
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.Optional; |
||||||
|
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
@Component |
||||||
|
public class TaskCacheEventHandler implements TaskEventHandler { |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private DataQualityResultOperator dataQualityResultOperator; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private ProcessService processService; |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private TaskInstanceDao taskInstanceDao; |
||||||
|
|
||||||
|
/** |
||||||
|
* handle CACHE task event |
||||||
|
* copy a new task instance from the cache task has been successfully run |
||||||
|
* @param taskEvent task event |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void handleTaskEvent(TaskEvent taskEvent) { |
||||||
|
int taskInstanceId = taskEvent.getTaskInstanceId(); |
||||||
|
int processInstanceId = taskEvent.getProcessInstanceId(); |
||||||
|
|
||||||
|
WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId( |
||||||
|
processInstanceId); |
||||||
|
Optional<TaskInstance> taskInstanceOptional = workflowExecuteRunnable.getTaskInstance(taskInstanceId); |
||||||
|
if (!taskInstanceOptional.isPresent()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
TaskInstance taskInstance = taskInstanceOptional.get(); |
||||||
|
dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance); |
||||||
|
|
||||||
|
TaskInstance cacheTaskInstance = taskInstanceDao.findTaskInstanceById(taskEvent.getCacheTaskInstanceId()); |
||||||
|
|
||||||
|
// keep the task instance fields
|
||||||
|
cacheTaskInstance.setId(taskInstance.getId()); |
||||||
|
cacheTaskInstance.setProcessInstanceId(processInstanceId); |
||||||
|
cacheTaskInstance.setProcessInstanceName(taskInstance.getProcessInstanceName()); |
||||||
|
cacheTaskInstance.setProcessInstance(taskInstance.getProcessInstance()); |
||||||
|
cacheTaskInstance.setProcessDefine(taskInstance.getProcessDefine()); |
||||||
|
cacheTaskInstance.setStartTime(taskInstance.getSubmitTime()); |
||||||
|
cacheTaskInstance.setSubmitTime(taskInstance.getSubmitTime()); |
||||||
|
cacheTaskInstance.setEndTime(new Date()); |
||||||
|
cacheTaskInstance.setFlag(Flag.YES); |
||||||
|
|
||||||
|
TaskInstanceUtils.copyTaskInstance(cacheTaskInstance, taskInstance); |
||||||
|
|
||||||
|
processService.changeOutParam(taskInstance); |
||||||
|
|
||||||
|
taskInstanceDao.updateTaskInstance(taskInstance); |
||||||
|
TaskStateEvent stateEvent = TaskStateEvent.builder() |
||||||
|
.processInstanceId(taskEvent.getProcessInstanceId()) |
||||||
|
.taskInstanceId(taskEvent.getTaskInstanceId()) |
||||||
|
.status(taskEvent.getState()) |
||||||
|
.type(StateEventType.TASK_STATE_CHANGE) |
||||||
|
.build(); |
||||||
|
|
||||||
|
workflowExecuteThreadPool.submitStateEvent(stateEvent); |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public TaskEventType getHandleEventType() { |
||||||
|
return TaskEventType.CACHE; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,112 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.event; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.Flag; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||||
|
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||||
|
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; |
||||||
|
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; |
||||||
|
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; |
||||||
|
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; |
||||||
|
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||||
|
import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; |
||||||
|
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
import java.util.HashMap; |
||||||
|
import java.util.Optional; |
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions; |
||||||
|
import org.junit.jupiter.api.Test; |
||||||
|
import org.junit.jupiter.api.extension.ExtendWith; |
||||||
|
import org.mockito.InjectMocks; |
||||||
|
import org.mockito.Mock; |
||||||
|
import org.mockito.Mockito; |
||||||
|
import org.mockito.junit.jupiter.MockitoExtension; |
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class) |
||||||
|
class TaskCacheEventHandlerTest { |
||||||
|
|
||||||
|
@InjectMocks |
||||||
|
private TaskCacheEventHandler taskCacheEventHandler; |
||||||
|
|
||||||
|
@Mock |
||||||
|
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; |
||||||
|
|
||||||
|
@Mock |
||||||
|
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||||
|
|
||||||
|
@Mock |
||||||
|
private DataQualityResultOperator dataQualityResultOperator; |
||||||
|
|
||||||
|
@Mock |
||||||
|
private ProcessService processService; |
||||||
|
|
||||||
|
@Mock |
||||||
|
private TaskInstanceDao taskInstanceDao; |
||||||
|
|
||||||
|
@Test |
||||||
|
void testHandleTaskEvent() { |
||||||
|
TaskEvent taskEvent = Mockito.mock(TaskEvent.class); |
||||||
|
int processInstanceId = 1; |
||||||
|
int taskInstanceId = 2; |
||||||
|
int cacheTaskInstanceId = 3; |
||||||
|
int cacheProcessInstanceId = 4; |
||||||
|
|
||||||
|
Mockito.when(taskEvent.getTaskInstanceId()).thenReturn(taskInstanceId); |
||||||
|
Mockito.when(taskEvent.getProcessInstanceId()).thenReturn(processInstanceId); |
||||||
|
Mockito.when(taskEvent.getCacheTaskInstanceId()).thenReturn(cacheTaskInstanceId); |
||||||
|
|
||||||
|
TaskInstance cacheTaskInstance = new TaskInstance(); |
||||||
|
cacheTaskInstance.setId(cacheTaskInstanceId); |
||||||
|
cacheTaskInstance.setProcessInstanceId(cacheProcessInstanceId); |
||||||
|
cacheTaskInstance.setTaskParams(JSONUtils.toJsonString(new HashMap<>())); |
||||||
|
|
||||||
|
Mockito.when(taskInstanceDao.findTaskInstanceById(cacheTaskInstanceId)).thenReturn(cacheTaskInstance); |
||||||
|
|
||||||
|
WorkflowExecuteRunnable workflowExecuteRunnable = Mockito.mock(WorkflowExecuteRunnable.class); |
||||||
|
Mockito.when(processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId)) |
||||||
|
.thenReturn(workflowExecuteRunnable); |
||||||
|
Optional<TaskInstance> taskInstanceOptional = Mockito.mock(Optional.class); |
||||||
|
Mockito.when(workflowExecuteRunnable.getTaskInstance(taskInstanceId)).thenReturn(taskInstanceOptional); |
||||||
|
Mockito.when(taskInstanceOptional.isPresent()).thenReturn(true); |
||||||
|
|
||||||
|
TaskInstance taskInstance = new TaskInstance(); |
||||||
|
taskInstance.setTaskParams(JSONUtils.toJsonString(new HashMap<>())); |
||||||
|
taskInstance.setId(taskInstanceId); |
||||||
|
taskInstance.setProcessInstanceId(processInstanceId); |
||||||
|
taskInstance.setProcessInstanceName("test"); |
||||||
|
ProcessInstance processInstance = new ProcessInstance(); |
||||||
|
taskInstance.setProcessInstance(processInstance); |
||||||
|
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||||
|
taskInstance.setProcessDefine(processDefinition); |
||||||
|
taskInstance.setSubmitTime(new Date()); |
||||||
|
|
||||||
|
Mockito.when(taskInstanceOptional.get()).thenReturn(taskInstance); |
||||||
|
|
||||||
|
taskCacheEventHandler.handleTaskEvent(taskEvent); |
||||||
|
|
||||||
|
Assertions.assertEquals(Flag.YES, taskInstance.getFlag()); |
||||||
|
Assertions.assertEquals(taskInstanceId, taskInstance.getId()); |
||||||
|
Assertions.assertEquals(processInstanceId, taskInstance.getProcessInstanceId()); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,29 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
import { useI18n } from 'vue-i18n' |
||||||
|
import type { IJsonItem } from '../types' |
||||||
|
|
||||||
|
export function useCache(): IJsonItem { |
||||||
|
const { t } = useI18n() |
||||||
|
return { |
||||||
|
type: 'switch', |
||||||
|
field: 'isCache', |
||||||
|
name: t('project.node.is_cache'), |
||||||
|
span: 12 |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue