Browse Source
* add task event thread * license heander * ci test * delete unuse file * fix CI test Co-authored-by: caishunfeng <534328519@qq.com>3.0.0/version-upgrade
caishunfeng
3 years ago
committed by
GitHub
4 changed files with 400 additions and 160 deletions
@ -0,0 +1,224 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.processor.queue; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Event; |
||||
import org.apache.dolphinscheduler.common.enums.StateEvent; |
||||
import org.apache.dolphinscheduler.common.enums.StateEventType; |
||||
import org.apache.dolphinscheduler.dao.entity.TaskInstance; |
||||
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand; |
||||
import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand; |
||||
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||
import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
|
||||
import java.util.concurrent.ConcurrentLinkedQueue; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
import io.netty.channel.Channel; |
||||
|
||||
/** |
||||
* task execute thread |
||||
*/ |
||||
public class TaskExecuteThread { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TaskExecuteThread.class); |
||||
|
||||
private final int processInstanceId; |
||||
|
||||
private final ConcurrentLinkedQueue<TaskEvent> events = new ConcurrentLinkedQueue<>(); |
||||
|
||||
private ProcessService processService; |
||||
|
||||
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||
|
||||
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; |
||||
|
||||
private DataQualityResultOperator dataQualityResultOperator; |
||||
|
||||
public TaskExecuteThread(int processInstanceId, ProcessService processService, WorkflowExecuteThreadPool workflowExecuteThreadPool, |
||||
ProcessInstanceExecCacheManager processInstanceExecCacheManager, DataQualityResultOperator dataQualityResultOperator) { |
||||
this.processInstanceId = processInstanceId; |
||||
this.processService = processService; |
||||
this.workflowExecuteThreadPool = workflowExecuteThreadPool; |
||||
this.processInstanceExecCacheManager = processInstanceExecCacheManager; |
||||
this.dataQualityResultOperator = dataQualityResultOperator; |
||||
} |
||||
|
||||
public void run() { |
||||
while (!this.events.isEmpty()) { |
||||
TaskEvent event = this.events.peek(); |
||||
try { |
||||
persist(event); |
||||
} catch (Exception e) { |
||||
logger.error("persist error, event:{}, error: {}", event, e); |
||||
} finally { |
||||
this.events.remove(event); |
||||
} |
||||
} |
||||
} |
||||
|
||||
public String getKey() { |
||||
return String.valueOf(processInstanceId); |
||||
} |
||||
|
||||
public int eventSize() { |
||||
return this.events.size(); |
||||
} |
||||
|
||||
public boolean isEmpty() { |
||||
return this.events.isEmpty(); |
||||
} |
||||
|
||||
public Integer getProcessInstanceId() { |
||||
return processInstanceId; |
||||
} |
||||
|
||||
public boolean addEvent(TaskEvent event) { |
||||
if (event.getProcessInstanceId() != this.processInstanceId) { |
||||
logger.warn("event would be abounded, task instance id:{}, process instance id:{}, this.processInstanceId:{}", |
||||
event.getTaskInstanceId(), event.getProcessInstanceId(), this.processInstanceId); |
||||
return false; |
||||
} |
||||
return this.events.add(event); |
||||
} |
||||
|
||||
/** |
||||
* persist task event |
||||
* |
||||
* @param taskEvent taskEvent |
||||
*/ |
||||
private void persist(TaskEvent taskEvent) { |
||||
Event event = taskEvent.getEvent(); |
||||
int taskInstanceId = taskEvent.getTaskInstanceId(); |
||||
int processInstanceId = taskEvent.getProcessInstanceId(); |
||||
|
||||
TaskInstance taskInstance; |
||||
WorkflowExecuteThread workflowExecuteThread = this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); |
||||
if (workflowExecuteThread != null && workflowExecuteThread.checkTaskInstanceById(taskInstanceId)) { |
||||
taskInstance = workflowExecuteThread.getTaskInstance(taskInstanceId); |
||||
} else { |
||||
taskInstance = processService.findTaskInstanceById(taskInstanceId); |
||||
} |
||||
|
||||
switch (event) { |
||||
case DISPATCH: |
||||
handleDispatchEvent(taskEvent, taskInstance); |
||||
// dispatch event do not need to submit state event
|
||||
return; |
||||
case DELAY: |
||||
case RUNNING: |
||||
handleRunningEvent(taskEvent, taskInstance); |
||||
break; |
||||
case RESULT: |
||||
handleResultEvent(taskEvent, taskInstance); |
||||
break; |
||||
default: |
||||
throw new IllegalArgumentException("invalid event type : " + event); |
||||
} |
||||
|
||||
StateEvent stateEvent = new StateEvent(); |
||||
stateEvent.setProcessInstanceId(taskEvent.getProcessInstanceId()); |
||||
stateEvent.setTaskInstanceId(taskEvent.getTaskInstanceId()); |
||||
stateEvent.setExecutionStatus(taskEvent.getState()); |
||||
stateEvent.setType(StateEventType.TASK_STATE_CHANGE); |
||||
workflowExecuteThreadPool.submitStateEvent(stateEvent); |
||||
} |
||||
|
||||
/** |
||||
* handle dispatch event |
||||
*/ |
||||
private void handleDispatchEvent(TaskEvent taskEvent, TaskInstance taskInstance) { |
||||
if (taskInstance == null) { |
||||
logger.error("taskInstance is null"); |
||||
return; |
||||
} |
||||
if (taskInstance.getState() != ExecutionStatus.SUBMITTED_SUCCESS) { |
||||
return; |
||||
} |
||||
taskInstance.setState(ExecutionStatus.DISPATCH); |
||||
taskInstance.setHost(taskEvent.getWorkerAddress()); |
||||
processService.saveTaskInstance(taskInstance); |
||||
} |
||||
|
||||
/** |
||||
* handle running event |
||||
*/ |
||||
private void handleRunningEvent(TaskEvent taskEvent, TaskInstance taskInstance) { |
||||
Channel channel = taskEvent.getChannel(); |
||||
try { |
||||
if (taskInstance != null) { |
||||
if (taskInstance.getState().typeIsFinished()) { |
||||
logger.warn("task is finish, running event is meaningless, taskInstanceId:{}, state:{}", taskInstance.getId(), taskInstance.getState()); |
||||
} else { |
||||
taskInstance.setState(taskEvent.getState()); |
||||
taskInstance.setStartTime(taskEvent.getStartTime()); |
||||
taskInstance.setHost(taskEvent.getWorkerAddress()); |
||||
taskInstance.setLogPath(taskEvent.getLogPath()); |
||||
taskInstance.setExecutePath(taskEvent.getExecutePath()); |
||||
taskInstance.setPid(taskEvent.getProcessId()); |
||||
taskInstance.setAppLink(taskEvent.getAppIds()); |
||||
processService.saveTaskInstance(taskInstance); |
||||
} |
||||
} |
||||
// if taskInstance is null (maybe deleted) or finish. retry will be meaningless . so ack success
|
||||
TaskExecuteRunningAckCommand taskExecuteRunningAckCommand = new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId()); |
||||
channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command()); |
||||
} catch (Exception e) { |
||||
logger.error("worker ack master error", e); |
||||
TaskExecuteRunningAckCommand taskExecuteRunningAckCommand = new TaskExecuteRunningAckCommand(ExecutionStatus.FAILURE.getCode(), -1); |
||||
channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command()); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* handle result event |
||||
*/ |
||||
private void handleResultEvent(TaskEvent taskEvent, TaskInstance taskInstance) { |
||||
Channel channel = taskEvent.getChannel(); |
||||
try { |
||||
if (taskInstance != null) { |
||||
dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance); |
||||
|
||||
taskInstance.setStartTime(taskEvent.getStartTime()); |
||||
taskInstance.setHost(taskEvent.getWorkerAddress()); |
||||
taskInstance.setLogPath(taskEvent.getLogPath()); |
||||
taskInstance.setExecutePath(taskEvent.getExecutePath()); |
||||
taskInstance.setPid(taskEvent.getProcessId()); |
||||
taskInstance.setAppLink(taskEvent.getAppIds()); |
||||
taskInstance.setState(taskEvent.getState()); |
||||
taskInstance.setEndTime(taskEvent.getEndTime()); |
||||
taskInstance.setVarPool(taskEvent.getVarPool()); |
||||
processService.changeOutParam(taskInstance); |
||||
processService.saveTaskInstance(taskInstance); |
||||
} |
||||
// if taskInstance is null (maybe deleted) . retry will be meaningless . so response success
|
||||
TaskExecuteResponseAckCommand taskExecuteResponseAckCommand = new TaskExecuteResponseAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId()); |
||||
channel.writeAndFlush(taskExecuteResponseAckCommand.convert2Command()); |
||||
} catch (Exception e) { |
||||
logger.error("worker response master error", e); |
||||
TaskExecuteResponseAckCommand taskExecuteResponseAckCommand = new TaskExecuteResponseAckCommand(ExecutionStatus.FAILURE.getCode(), -1); |
||||
channel.writeAndFlush(taskExecuteResponseAckCommand.convert2Command()); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,138 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.processor.queue; |
||||
|
||||
import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; |
||||
import org.apache.dolphinscheduler.server.master.config.MasterConfig; |
||||
import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; |
||||
import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
|
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
|
||||
import javax.annotation.PostConstruct; |
||||
|
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; |
||||
import org.springframework.stereotype.Component; |
||||
import org.springframework.util.concurrent.ListenableFuture; |
||||
import org.springframework.util.concurrent.ListenableFutureCallback; |
||||
|
||||
@Component |
||||
public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor { |
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(TaskExecuteThreadPool.class); |
||||
|
||||
private final ConcurrentHashMap<String, TaskExecuteThread> multiThreadFilterMap = new ConcurrentHashMap<>(); |
||||
|
||||
@Autowired |
||||
private MasterConfig masterConfig; |
||||
|
||||
@Autowired |
||||
private ProcessInstanceExecCacheManager processInstanceExecCacheManager; |
||||
|
||||
/** |
||||
* process service |
||||
*/ |
||||
@Autowired |
||||
private ProcessService processService; |
||||
|
||||
/** |
||||
* data quality result operator |
||||
*/ |
||||
@Autowired |
||||
private DataQualityResultOperator dataQualityResultOperator; |
||||
|
||||
|
||||
@Autowired |
||||
private WorkflowExecuteThreadPool workflowExecuteThreadPool; |
||||
|
||||
/** |
||||
* task event thread map |
||||
*/ |
||||
private final ConcurrentHashMap<Integer, TaskExecuteThread> taskExecuteThreadMap = new ConcurrentHashMap<>(); |
||||
|
||||
@PostConstruct |
||||
private void init() { |
||||
this.setDaemon(true); |
||||
this.setThreadNamePrefix("Task-Execute-Thread-"); |
||||
this.setMaxPoolSize(masterConfig.getExecThreads()); |
||||
this.setCorePoolSize(masterConfig.getExecThreads()); |
||||
} |
||||
|
||||
public void submitTaskEvent(TaskEvent taskEvent) { |
||||
if (!processInstanceExecCacheManager.contains(taskEvent.getProcessInstanceId())) { |
||||
logger.warn("workflowExecuteThread is null, event: {}", taskEvent); |
||||
return; |
||||
} |
||||
if (!taskExecuteThreadMap.containsKey(taskEvent.getProcessInstanceId())) { |
||||
TaskExecuteThread taskExecuteThread = new TaskExecuteThread( |
||||
taskEvent.getProcessInstanceId(), |
||||
processService, workflowExecuteThreadPool, |
||||
processInstanceExecCacheManager, |
||||
dataQualityResultOperator); |
||||
taskExecuteThreadMap.put(taskEvent.getProcessInstanceId(), taskExecuteThread); |
||||
} |
||||
TaskExecuteThread taskExecuteThread = taskExecuteThreadMap.get(taskEvent.getProcessInstanceId()); |
||||
if (taskExecuteThread != null) { |
||||
taskExecuteThread.addEvent(taskEvent); |
||||
} |
||||
} |
||||
|
||||
public void eventHandler() { |
||||
for (TaskExecuteThread taskExecuteThread: taskExecuteThreadMap.values()) { |
||||
executeEvent(taskExecuteThread); |
||||
} |
||||
} |
||||
|
||||
public void executeEvent(TaskExecuteThread taskExecuteThread) { |
||||
if (taskExecuteThread.eventSize() == 0) { |
||||
return; |
||||
} |
||||
if (multiThreadFilterMap.containsKey(taskExecuteThread.getKey())) { |
||||
return; |
||||
} |
||||
ListenableFuture future = this.submitListenable(() -> { |
||||
taskExecuteThread.run(); |
||||
multiThreadFilterMap.put(taskExecuteThread.getKey(), taskExecuteThread); |
||||
}); |
||||
future.addCallback(new ListenableFutureCallback() { |
||||
@Override |
||||
public void onFailure(Throwable ex) { |
||||
logger.error("handle event {} failed: {}", taskExecuteThread.getProcessInstanceId(), ex); |
||||
if (!processInstanceExecCacheManager.contains(taskExecuteThread.getProcessInstanceId())) { |
||||
taskExecuteThreadMap.remove(taskExecuteThread.getProcessInstanceId()); |
||||
logger.info("remove process instance: {}", taskExecuteThread.getProcessInstanceId()); |
||||
} |
||||
multiThreadFilterMap.remove(taskExecuteThread.getKey()); |
||||
} |
||||
|
||||
@Override |
||||
public void onSuccess(Object result) { |
||||
logger.info("persist events {} succeeded.", taskExecuteThread.getProcessInstanceId()); |
||||
if (!processInstanceExecCacheManager.contains(taskExecuteThread.getProcessInstanceId())) { |
||||
taskExecuteThreadMap.remove(taskExecuteThread.getProcessInstanceId()); |
||||
logger.info("remove process instance: {}", taskExecuteThread.getProcessInstanceId()); |
||||
} |
||||
multiThreadFilterMap.remove(taskExecuteThread.getKey()); |
||||
} |
||||
}); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue