Browse Source
* Supports task instance cache operation * add task plugin cache * use SHA-256 to generate key * Update dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql Co-authored-by: Jay Chung <zhongjiajie955@gmail.com> * Update dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql Co-authored-by: Jay Chung <zhongjiajie955@gmail.com> * Optimizing database Scripts * Optimize clear cache operation Co-authored-by: Jay Chung <zhongjiajie955@gmail.com>3.2.0-release
JieguangZhou
2 years ago
committed by
GitHub
77 changed files with 1151 additions and 24 deletions
@ -0,0 +1,44 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.api.dto.taskInstance; |
||||
|
||||
import org.apache.dolphinscheduler.api.utils.Result; |
||||
|
||||
import lombok.Data; |
||||
|
||||
/** |
||||
* task instance success response |
||||
*/ |
||||
@Data |
||||
public class TaskInstanceRemoveCacheResponse extends Result { |
||||
|
||||
private String cacheKey; |
||||
|
||||
public TaskInstanceRemoveCacheResponse(Result result) { |
||||
super(); |
||||
this.setCode(result.getCode()); |
||||
this.setMsg(result.getMsg()); |
||||
} |
||||
|
||||
public TaskInstanceRemoveCacheResponse(Result result, String cacheKey) { |
||||
super(); |
||||
this.setCode(result.getCode()); |
||||
this.setMsg(result.getMsg()); |
||||
this.cacheKey = cacheKey; |
||||
} |
||||
} |
||||
@ -0,0 +1,161 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils; |
||||
import org.apache.commons.lang3.StringUtils; |
||||
import org.apache.commons.lang3.tuple.Pair; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.Comparator; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
import java.util.Set; |
||||
import java.util.regex.Matcher; |
||||
import java.util.regex.Pattern; |
||||
import java.util.stream.Collectors; |
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode; |
||||
|
||||
public class TaskCacheUtils { |
||||
|
||||
private TaskCacheUtils() { |
||||
throw new IllegalStateException("Utility class"); |
||||
} |
||||
|
||||
public static final String MERGE_TAG = "-"; |
||||
|
||||
/** |
||||
* generate cache key for task instance |
||||
* the follow message will be used to generate cache key |
||||
* 2. task version |
||||
* 3. task is cache |
||||
* 4. input VarPool, from upstream task and workflow global parameters |
||||
* @param taskInstance task instance |
||||
* @param taskExecutionContext taskExecutionContext |
||||
* @return cache key |
||||
*/ |
||||
public static String generateCacheKey(TaskInstance taskInstance, TaskExecutionContext taskExecutionContext) { |
||||
List<String> keyElements = new ArrayList<>(); |
||||
keyElements.add(String.valueOf(taskInstance.getTaskCode())); |
||||
keyElements.add(String.valueOf(taskInstance.getTaskDefinitionVersion())); |
||||
keyElements.add(String.valueOf(taskInstance.getIsCache().getCode())); |
||||
keyElements.add(String.valueOf(taskInstance.getEnvironmentConfig())); |
||||
keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext)); |
||||
String data = StringUtils.join(keyElements, "_"); |
||||
return DigestUtils.sha256Hex(data); |
||||
} |
||||
|
||||
/** |
||||
* generate cache key for task instance which is cache execute |
||||
* this key will record which cache task instance will be copied, and cache key will be used |
||||
* tagCacheKey = sourceTaskId + "-" + cacheKey |
||||
* @param sourceTaskId source task id |
||||
* @param cacheKey cache key |
||||
* @return tagCacheKey |
||||
*/ |
||||
public static String generateTagCacheKey(Integer sourceTaskId, String cacheKey) { |
||||
return sourceTaskId + MERGE_TAG + cacheKey; |
||||
} |
||||
|
||||
/** |
||||
* revert cache key tag to source task id and cache key |
||||
* @param tagCacheKey cache key |
||||
* @return Pair<Integer, String>, first is source task id, second is cache key |
||||
*/ |
||||
public static Pair<Integer, String> revertCacheKey(String tagCacheKey) { |
||||
Pair<Integer, String> taskIdAndCacheKey; |
||||
if (tagCacheKey == null) { |
||||
taskIdAndCacheKey = Pair.of(-1, ""); |
||||
return taskIdAndCacheKey; |
||||
} |
||||
if (tagCacheKey.contains(MERGE_TAG)) { |
||||
String[] split = tagCacheKey.split(MERGE_TAG); |
||||
if (split.length == 2) { |
||||
taskIdAndCacheKey = Pair.of(Integer.parseInt(split[0]), split[1]); |
||||
} else { |
||||
taskIdAndCacheKey = Pair.of(-1, ""); |
||||
} |
||||
return taskIdAndCacheKey; |
||||
} else { |
||||
return Pair.of(-1, tagCacheKey); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* get hash data of task input var pool |
||||
* there are two parts of task input var pool: from upstream task and workflow global parameters |
||||
* @param taskInstance task instance |
||||
* taskExecutionContext taskExecutionContext |
||||
*/ |
||||
public static String getTaskInputVarPoolData(TaskInstance taskInstance, TaskExecutionContext context) { |
||||
JsonNode taskParams = JSONUtils.parseObject(taskInstance.getTaskParams()); |
||||
|
||||
// The set of input values considered from localParams in the taskParams
|
||||
Set<String> propertyInSet = JSONUtils.toList(taskParams.get("localParams").toString(), Property.class).stream() |
||||
.filter(property -> property.getDirect().equals(Direct.IN)) |
||||
.map(Property::getProp).collect(Collectors.toSet()); |
||||
|
||||
// The set of input values considered from `${var}` form task definition
|
||||
propertyInSet.addAll(getScriptVarInSet(taskInstance)); |
||||
|
||||
// var pool value from upstream task
|
||||
List<Property> varPool = JSONUtils.toList(taskInstance.getVarPool(), Property.class); |
||||
|
||||
// var pool value from workflow global parameters
|
||||
if (context.getPrepareParamsMap() != null) { |
||||
Set<String> taskVarPoolSet = varPool.stream().map(Property::getProp).collect(Collectors.toSet()); |
||||
List<Property> globalContextVarPool = context.getPrepareParamsMap().entrySet().stream() |
||||
.filter(entry -> !taskVarPoolSet.contains(entry.getKey())) |
||||
.map(Map.Entry::getValue) |
||||
.collect(Collectors.toList()); |
||||
varPool.addAll(globalContextVarPool); |
||||
} |
||||
|
||||
// only consider var pool value which is in propertyInSet
|
||||
varPool = varPool.stream() |
||||
.filter(property -> property.getDirect().equals(Direct.IN)) |
||||
.filter(property -> propertyInSet.contains(property.getProp())) |
||||
.sorted(Comparator.comparing(Property::getProp)) |
||||
.collect(Collectors.toList()); |
||||
return JSONUtils.toJsonString(varPool); |
||||
} |
||||
|
||||
/** |
||||
* get var in set from task definition |
||||
* @param taskInstance task instance |
||||
* @return var in set |
||||
*/ |
||||
public static List<String> getScriptVarInSet(TaskInstance taskInstance) { |
||||
Pattern pattern = Pattern.compile("\\$\\{(.+?)\\}"); |
||||
Matcher matcher = pattern.matcher(taskInstance.getTaskParams()); |
||||
|
||||
List<String> varInSet = new ArrayList<>(); |
||||
while (matcher.find()) { |
||||
varInSet.add(matcher.group(1)); |
||||
} |
||||
return varInSet; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,172 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.dao.utils; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Flag; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; |
||||
import org.apache.dolphinscheduler.plugin.task.api.model.Property; |
||||
|
||||
import org.apache.commons.lang3.tuple.Pair; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.Arrays; |
||||
import java.util.HashMap; |
||||
import java.util.List; |
||||
import java.util.Map; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.BeforeEach; |
||||
import org.junit.jupiter.api.Test; |
||||
|
||||
class TaskCacheUtilsTest { |
||||
|
||||
private TaskInstance taskInstance; |
||||
|
||||
private TaskExecutionContext taskExecutionContext; |
||||
|
||||
@BeforeEach |
||||
void setUp() { |
||||
String taskParams = "{\n" + |
||||
" \"localParams\": [\n" + |
||||
" {\n" + |
||||
" \"prop\": \"a\",\n" + |
||||
" \"direct\": \"IN\",\n" + |
||||
" \"type\": \"VARCHAR\",\n" + |
||||
" \"value\": \"\"\n" + |
||||
" },\n" + |
||||
" {\n" + |
||||
" \"prop\": \"b\",\n" + |
||||
" \"direct\": \"IN\",\n" + |
||||
" \"type\": \"VARCHAR\",\n" + |
||||
" \"value\": \"bb\"\n" + |
||||
" }\n" + |
||||
" ],\n" + |
||||
" \"rawScript\": \"echo ${c}\\necho ${d}\",\n" + |
||||
" \"resourceList\": []\n" + |
||||
"}"; |
||||
|
||||
String varPool = "[\n" + |
||||
" {\n" + |
||||
" \"prop\": \"c\",\n" + |
||||
" \"direct\": \"IN\",\n" + |
||||
" \"type\": \"VARCHAR\",\n" + |
||||
" \"value\": \"cc\"\n" + |
||||
" },\n" + |
||||
" {\n" + |
||||
" \"prop\": \"k\",\n" + |
||||
" \"direct\": \"IN\",\n" + |
||||
" \"type\": \"VARCHAR\",\n" + |
||||
" \"value\": \"kk\"\n" + |
||||
" }\n" + |
||||
"]"; |
||||
|
||||
taskInstance = new TaskInstance(); |
||||
taskInstance.setTaskParams(taskParams); |
||||
taskInstance.setVarPool(varPool); |
||||
taskInstance.setTaskCode(123L); |
||||
taskInstance.setTaskDefinitionVersion(1); |
||||
taskInstance.setIsCache(Flag.YES); |
||||
|
||||
taskExecutionContext = new TaskExecutionContext(); |
||||
Property property = new Property(); |
||||
property.setProp("a"); |
||||
property.setDirect(Direct.IN); |
||||
property.setType(DataType.VARCHAR); |
||||
property.setValue("aa"); |
||||
Map<String, Property> prepareParamsMap = new HashMap<>(); |
||||
prepareParamsMap.put("a", property); |
||||
taskExecutionContext.setPrepareParamsMap(prepareParamsMap); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
void testRevertCacheKey() { |
||||
Pair<Integer, String> taskIdAndCacheKey1 = TaskCacheUtils.revertCacheKey(null); |
||||
Assertions.assertEquals(Pair.of(-1, ""), taskIdAndCacheKey1); |
||||
|
||||
Pair<Integer, String> taskIdAndCacheKey2 = TaskCacheUtils.revertCacheKey("123"); |
||||
Assertions.assertEquals(Pair.of(-1, "123"), taskIdAndCacheKey2); |
||||
|
||||
Pair<Integer, String> taskIdAndCacheKey3 = TaskCacheUtils.revertCacheKey("1-123"); |
||||
Assertions.assertEquals(Pair.of(1, "123"), taskIdAndCacheKey3); |
||||
|
||||
Pair<Integer, String> taskIdAndCacheKey4 = TaskCacheUtils.revertCacheKey("1-123-4"); |
||||
Assertions.assertEquals(Pair.of(-1, ""), taskIdAndCacheKey4); |
||||
} |
||||
|
||||
@Test |
||||
void testGetScriptVarInSet() { |
||||
List<String> scriptVarInSet = TaskCacheUtils.getScriptVarInSet(taskInstance); |
||||
List<String> except = new ArrayList<>(Arrays.asList("c", "d")); |
||||
Assertions.assertEquals(except, scriptVarInSet); |
||||
} |
||||
|
||||
@Test |
||||
void TestGetTaskInputVarPoolData() { |
||||
TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext); |
||||
// only a=aa and c=cc will influence the result,
|
||||
// b=bb is a fixed value, will be considered in task version
|
||||
// k=kk is not in task params, will be ignored
|
||||
String except = |
||||
"[{\"prop\":\"a\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"aa\"},{\"prop\":\"c\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"cc\"}]"; |
||||
Assertions.assertEquals(except, TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext)); |
||||
} |
||||
|
||||
@Test |
||||
void TestGenerateCacheKey() { |
||||
String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||
Property propertyI = new Property(); |
||||
propertyI.setProp("i"); |
||||
propertyI.setDirect(Direct.IN); |
||||
propertyI.setType(DataType.VARCHAR); |
||||
propertyI.setValue("ii"); |
||||
taskExecutionContext.getPrepareParamsMap().put("i", propertyI); |
||||
String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||
// i will not influence the result, because task instance not use it
|
||||
Assertions.assertEquals(cacheKeyBase, cacheKeyNew); |
||||
|
||||
Property propertyD = new Property(); |
||||
propertyD.setProp("d"); |
||||
propertyD.setDirect(Direct.IN); |
||||
propertyD.setType(DataType.VARCHAR); |
||||
propertyD.setValue("dd"); |
||||
taskExecutionContext.getPrepareParamsMap().put("i", propertyD); |
||||
String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||
// d will influence the result, because task instance use it
|
||||
Assertions.assertNotEquals(cacheKeyBase, cacheKeyD); |
||||
|
||||
taskInstance.setTaskDefinitionVersion(100); |
||||
String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||
// task definition version is changed, so cache key changed
|
||||
Assertions.assertNotEquals(cacheKeyD, cacheKeyE); |
||||
|
||||
taskInstance.setEnvironmentConfig("export PYTHON_HOME=/bin/python3"); |
||||
String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext); |
||||
// EnvironmentConfig is changed, so cache key changed
|
||||
Assertions.assertNotEquals(cacheKeyE, cacheKeyF); |
||||
} |
||||
|
||||
@Test |
||||
void testGetCacheKey() { |
||||
String cacheKey = TaskCacheUtils.generateTagCacheKey(1, "123"); |
||||
Assertions.assertEquals("1-123", cacheKey); |
||||
} |
||||
} |
||||
@ -0,0 +1,109 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.event; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Flag; |
||||
import org.apache.dolphinscheduler.common.enums.StateEventType; |
||||
import org.apache.dolphinscheduler.common.enums.TaskEventType; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; |
||||
import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils; |
||||
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; |
||||
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||
import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
|
||||
import java.util.Date; |
||||
import java.util.Optional; |
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
@Component |
||||
public class TaskCacheEventHandler implements TaskEventHandler { |
||||
|
||||
@Autowired |
||||
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; |
||||
|
||||
@Autowired |
||||
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||
|
||||
@Autowired |
||||
private DataQualityResultOperator dataQualityResultOperator; |
||||
|
||||
@Autowired |
||||
private ProcessService processService; |
||||
|
||||
@Autowired |
||||
private TaskInstanceDao taskInstanceDao; |
||||
|
||||
/** |
||||
* handle CACHE task event |
||||
* copy a new task instance from the cache task has been successfully run |
||||
* @param taskEvent task event |
||||
*/ |
||||
@Override |
||||
public void handleTaskEvent(TaskEvent taskEvent) { |
||||
int taskInstanceId = taskEvent.getTaskInstanceId(); |
||||
int processInstanceId = taskEvent.getProcessInstanceId(); |
||||
|
||||
WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId( |
||||
processInstanceId); |
||||
Optional<TaskInstance> taskInstanceOptional = workflowExecuteRunnable.getTaskInstance(taskInstanceId); |
||||
if (!taskInstanceOptional.isPresent()) { |
||||
return; |
||||
} |
||||
TaskInstance taskInstance = taskInstanceOptional.get(); |
||||
dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance); |
||||
|
||||
TaskInstance cacheTaskInstance = taskInstanceDao.findTaskInstanceById(taskEvent.getCacheTaskInstanceId()); |
||||
|
||||
// keep the task instance fields
|
||||
cacheTaskInstance.setId(taskInstance.getId()); |
||||
cacheTaskInstance.setProcessInstanceId(processInstanceId); |
||||
cacheTaskInstance.setProcessInstanceName(taskInstance.getProcessInstanceName()); |
||||
cacheTaskInstance.setProcessInstance(taskInstance.getProcessInstance()); |
||||
cacheTaskInstance.setProcessDefine(taskInstance.getProcessDefine()); |
||||
cacheTaskInstance.setStartTime(taskInstance.getSubmitTime()); |
||||
cacheTaskInstance.setSubmitTime(taskInstance.getSubmitTime()); |
||||
cacheTaskInstance.setEndTime(new Date()); |
||||
cacheTaskInstance.setFlag(Flag.YES); |
||||
|
||||
TaskInstanceUtils.copyTaskInstance(cacheTaskInstance, taskInstance); |
||||
|
||||
processService.changeOutParam(taskInstance); |
||||
|
||||
taskInstanceDao.updateTaskInstance(taskInstance); |
||||
TaskStateEvent stateEvent = TaskStateEvent.builder() |
||||
.processInstanceId(taskEvent.getProcessInstanceId()) |
||||
.taskInstanceId(taskEvent.getTaskInstanceId()) |
||||
.status(taskEvent.getState()) |
||||
.type(StateEventType.TASK_STATE_CHANGE) |
||||
.build(); |
||||
|
||||
workflowExecuteThreadPool.submitStateEvent(stateEvent); |
||||
|
||||
} |
||||
|
||||
@Override |
||||
public TaskEventType getHandleEventType() { |
||||
return TaskEventType.CACHE; |
||||
} |
||||
} |
||||
@ -0,0 +1,112 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.event; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Flag; |
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; |
||||
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; |
||||
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; |
||||
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||
import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
|
||||
import java.util.Date; |
||||
import java.util.HashMap; |
||||
import java.util.Optional; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.InjectMocks; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
class TaskCacheEventHandlerTest { |
||||
|
||||
@InjectMocks |
||||
private TaskCacheEventHandler taskCacheEventHandler; |
||||
|
||||
@Mock |
||||
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; |
||||
|
||||
@Mock |
||||
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||
|
||||
@Mock |
||||
private DataQualityResultOperator dataQualityResultOperator; |
||||
|
||||
@Mock |
||||
private ProcessService processService; |
||||
|
||||
@Mock |
||||
private TaskInstanceDao taskInstanceDao; |
||||
|
||||
@Test |
||||
void testHandleTaskEvent() { |
||||
TaskEvent taskEvent = Mockito.mock(TaskEvent.class); |
||||
int processInstanceId = 1; |
||||
int taskInstanceId = 2; |
||||
int cacheTaskInstanceId = 3; |
||||
int cacheProcessInstanceId = 4; |
||||
|
||||
Mockito.when(taskEvent.getTaskInstanceId()).thenReturn(taskInstanceId); |
||||
Mockito.when(taskEvent.getProcessInstanceId()).thenReturn(processInstanceId); |
||||
Mockito.when(taskEvent.getCacheTaskInstanceId()).thenReturn(cacheTaskInstanceId); |
||||
|
||||
TaskInstance cacheTaskInstance = new TaskInstance(); |
||||
cacheTaskInstance.setId(cacheTaskInstanceId); |
||||
cacheTaskInstance.setProcessInstanceId(cacheProcessInstanceId); |
||||
cacheTaskInstance.setTaskParams(JSONUtils.toJsonString(new HashMap<>())); |
||||
|
||||
Mockito.when(taskInstanceDao.findTaskInstanceById(cacheTaskInstanceId)).thenReturn(cacheTaskInstance); |
||||
|
||||
WorkflowExecuteRunnable workflowExecuteRunnable = Mockito.mock(WorkflowExecuteRunnable.class); |
||||
Mockito.when(processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId)) |
||||
.thenReturn(workflowExecuteRunnable); |
||||
Optional<TaskInstance> taskInstanceOptional = Mockito.mock(Optional.class); |
||||
Mockito.when(workflowExecuteRunnable.getTaskInstance(taskInstanceId)).thenReturn(taskInstanceOptional); |
||||
Mockito.when(taskInstanceOptional.isPresent()).thenReturn(true); |
||||
|
||||
TaskInstance taskInstance = new TaskInstance(); |
||||
taskInstance.setTaskParams(JSONUtils.toJsonString(new HashMap<>())); |
||||
taskInstance.setId(taskInstanceId); |
||||
taskInstance.setProcessInstanceId(processInstanceId); |
||||
taskInstance.setProcessInstanceName("test"); |
||||
ProcessInstance processInstance = new ProcessInstance(); |
||||
taskInstance.setProcessInstance(processInstance); |
||||
ProcessDefinition processDefinition = new ProcessDefinition(); |
||||
taskInstance.setProcessDefine(processDefinition); |
||||
taskInstance.setSubmitTime(new Date()); |
||||
|
||||
Mockito.when(taskInstanceOptional.get()).thenReturn(taskInstance); |
||||
|
||||
taskCacheEventHandler.handleTaskEvent(taskEvent); |
||||
|
||||
Assertions.assertEquals(Flag.YES, taskInstance.getFlag()); |
||||
Assertions.assertEquals(taskInstanceId, taskInstance.getId()); |
||||
Assertions.assertEquals(processInstanceId, taskInstance.getProcessInstanceId()); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,29 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
import { useI18n } from 'vue-i18n' |
||||
import type { IJsonItem } from '../types' |
||||
|
||||
export function useCache(): IJsonItem { |
||||
const { t } = useI18n() |
||||
return { |
||||
type: 'switch', |
||||
field: 'isCache', |
||||
name: t('project.node.is_cache'), |
||||
span: 12 |
||||
} |
||||
} |
||||
Loading…
Reference in new issue