Browse Source
* 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * add license * modify javadoc error * TaskExecutionContext create modify * buildAckCommand taskInstanceId not set modify * java doc error modify * add comment * ExecutorManager interface add generic type * add TaskInstanceCacheManager receive Worker report result * TaskInstance setExecutePath * add TaskInstanceCacheManager to receive Worker Task result report * TaskInstanceCacheManager add remove method * add license * add dispatcht task method * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * 1，worker TaskPros use TaskExecutionContext replase 2，Master kill Task , KillTaskProcessor modify * worker remove db * ShellTask modify * master persistence processId and appIds * master persistence processId and appIds * master add kill task logic * master add kill task logic * master add kill task logic * javadoc error modify * remove chinese log * executeDirectly method add Override * remote module modify * TaskKillResponseProcessor command type modify * create buildKillCommand * host add host:port format * host add host:port format * TaskAckProcessor modify * TaskAckProcessor modify * task prioriry refator * remove ITaskQueue * task prioriry refator * remove ITaskQueue * TaskPriority refactor * remove logs * WorkerServer refactor * MasterSchedulerService modify * WorkerConfig listen port modify * modify master and worker listen port * cancelTaskInstance set TaskExecutionContext host,logPath,executePath * cancelTaskInstance set TaskExecutionContext host,logPath,executePath * Encapsulate the parameters required by sqltask * 1，Encapsulate the parameters required by sqltask 2，SQLTask optimization * AbstractTask modify * ProcedureTask optimization * MasterSchedulerService modify * TaskUpdateQueueConsumer modify * test * DataxTask process run debug * DataxTask process run debug * add protobuf dependency，MR、Spark task etc need this * TaskUpdateQueueConsumer modify * TaskExecutionContextBuilder set TaskInstance workgroup * WorkerGroupService queryAllGroup modify query available work group * 1，get workergroup from zk modify 2，SpringConnectionFactory repeat load modify * master and worker register ip use OSUtils.getHost() * ProcessInstance host set ip:port format * worker fault tolerance modify * Constants and .env modify * master fault tolerant bug modify * UT add pom.xml * timing online modify * when taskResponse is faster than taskAck to db，task state will error add async queue and new a thread reslove this problem * TaskExecutionContext set host Co-authored-by: qiaozhanwei <qiaozhanwei@analysys.com.cn>pull/2/head
qiaozhanwei
5 years ago
committed by
GitHub
5 changed files with 373 additions and 11 deletions
@ -0,0 +1,227 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.manager; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||||
|
|
||||||
|
import java.util.Date; |
||||||
|
|
||||||
|
/** |
||||||
|
* task event |
||||||
|
*/ |
||||||
|
public class TaskEvent { |
||||||
|
|
||||||
|
public static final String ACK = "ack"; |
||||||
|
public static final String RESPONSE = "response"; |
||||||
|
|
||||||
|
/** |
||||||
|
* taskInstanceId |
||||||
|
*/ |
||||||
|
private int taskInstanceId; |
||||||
|
|
||||||
|
/** |
||||||
|
* worker address |
||||||
|
*/ |
||||||
|
private String workerAddress; |
||||||
|
|
||||||
|
/** |
||||||
|
* state |
||||||
|
*/ |
||||||
|
private ExecutionStatus state; |
||||||
|
|
||||||
|
/** |
||||||
|
* start time |
||||||
|
*/ |
||||||
|
private Date startTime; |
||||||
|
|
||||||
|
/** |
||||||
|
* end time |
||||||
|
*/ |
||||||
|
private Date endTime; |
||||||
|
|
||||||
|
/** |
||||||
|
* execute path |
||||||
|
*/ |
||||||
|
private String executePath; |
||||||
|
|
||||||
|
/** |
||||||
|
* log path |
||||||
|
*/ |
||||||
|
private String logPath; |
||||||
|
|
||||||
|
/** |
||||||
|
* processId |
||||||
|
*/ |
||||||
|
private int processId; |
||||||
|
|
||||||
|
/** |
||||||
|
* appIds |
||||||
|
*/ |
||||||
|
private String appIds; |
||||||
|
|
||||||
|
/** |
||||||
|
* ack / response |
||||||
|
*/ |
||||||
|
private String type; |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* receive ack info |
||||||
|
* @param state state |
||||||
|
* @param startTime startTime |
||||||
|
* @param workerAddress workerAddress |
||||||
|
* @param executePath executePath |
||||||
|
* @param logPath logPath |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @param type type |
||||||
|
*/ |
||||||
|
public void receiveAck(ExecutionStatus state, |
||||||
|
Date startTime, |
||||||
|
String workerAddress, |
||||||
|
String executePath, |
||||||
|
String logPath, |
||||||
|
int taskInstanceId, |
||||||
|
String type){ |
||||||
|
this.state = state; |
||||||
|
this.startTime = startTime; |
||||||
|
this.workerAddress = workerAddress; |
||||||
|
this.executePath = executePath; |
||||||
|
this.logPath = logPath; |
||||||
|
this.taskInstanceId = taskInstanceId; |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* receive response info |
||||||
|
* @param state state |
||||||
|
* @param endTime endTime |
||||||
|
* @param processId processId |
||||||
|
* @param appIds appIds |
||||||
|
* @param taskInstanceId taskInstanceId |
||||||
|
* @param type type |
||||||
|
*/ |
||||||
|
public void receiveResponse(ExecutionStatus state, |
||||||
|
Date endTime, |
||||||
|
int processId, |
||||||
|
String appIds, |
||||||
|
int taskInstanceId, |
||||||
|
String type){ |
||||||
|
this.state = state; |
||||||
|
this.endTime = endTime; |
||||||
|
this.processId = processId; |
||||||
|
this.appIds = appIds; |
||||||
|
this.taskInstanceId = taskInstanceId; |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
public int getTaskInstanceId() { |
||||||
|
return taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTaskInstanceId(int taskInstanceId) { |
||||||
|
this.taskInstanceId = taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public String getWorkerAddress() { |
||||||
|
return workerAddress; |
||||||
|
} |
||||||
|
|
||||||
|
public void setWorkerAddress(String workerAddress) { |
||||||
|
this.workerAddress = workerAddress; |
||||||
|
} |
||||||
|
|
||||||
|
public ExecutionStatus getState() { |
||||||
|
return state; |
||||||
|
} |
||||||
|
|
||||||
|
public void setState(ExecutionStatus state) { |
||||||
|
this.state = state; |
||||||
|
} |
||||||
|
|
||||||
|
public Date getStartTime() { |
||||||
|
return startTime; |
||||||
|
} |
||||||
|
|
||||||
|
public void setStartTime(Date startTime) { |
||||||
|
this.startTime = startTime; |
||||||
|
} |
||||||
|
|
||||||
|
public Date getEndTime() { |
||||||
|
return endTime; |
||||||
|
} |
||||||
|
|
||||||
|
public void setEndTime(Date endTime) { |
||||||
|
this.endTime = endTime; |
||||||
|
} |
||||||
|
|
||||||
|
public String getExecutePath() { |
||||||
|
return executePath; |
||||||
|
} |
||||||
|
|
||||||
|
public void setExecutePath(String executePath) { |
||||||
|
this.executePath = executePath; |
||||||
|
} |
||||||
|
|
||||||
|
public String getLogPath() { |
||||||
|
return logPath; |
||||||
|
} |
||||||
|
|
||||||
|
public void setLogPath(String logPath) { |
||||||
|
this.logPath = logPath; |
||||||
|
} |
||||||
|
|
||||||
|
public int getProcessId() { |
||||||
|
return processId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setProcessId(int processId) { |
||||||
|
this.processId = processId; |
||||||
|
} |
||||||
|
|
||||||
|
public String getAppIds() { |
||||||
|
return appIds; |
||||||
|
} |
||||||
|
|
||||||
|
public void setAppIds(String appIds) { |
||||||
|
this.appIds = appIds; |
||||||
|
} |
||||||
|
|
||||||
|
public String getType() { |
||||||
|
return type; |
||||||
|
} |
||||||
|
|
||||||
|
public void setType(String type) { |
||||||
|
this.type = type; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "TaskEvent{" + |
||||||
|
"taskInstanceId=" + taskInstanceId + |
||||||
|
", workerAddress='" + workerAddress + '\'' + |
||||||
|
", state=" + state + |
||||||
|
", startTime=" + startTime + |
||||||
|
", endTime=" + endTime + |
||||||
|
", executePath='" + executePath + '\'' + |
||||||
|
", logPath='" + logPath + '\'' + |
||||||
|
", processId=" + processId + |
||||||
|
", appIds='" + appIds + '\'' + |
||||||
|
", type='" + type + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,121 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.manager; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.Constants; |
||||||
|
import org.apache.dolphinscheduler.common.thread.Stopper; |
||||||
|
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
import javax.annotation.PostConstruct; |
||||||
|
import java.util.concurrent.BlockingQueue; |
||||||
|
import java.util.concurrent.LinkedBlockingQueue; |
||||||
|
|
||||||
|
/** |
||||||
|
* task manager |
||||||
|
*/ |
||||||
|
@Component |
||||||
|
public class TaskManager { |
||||||
|
|
||||||
|
/** |
||||||
|
* logger |
||||||
|
*/ |
||||||
|
private static final Logger logger = LoggerFactory.getLogger(TaskManager.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* attemptQueue |
||||||
|
*/ |
||||||
|
private final BlockingQueue<TaskEvent> attemptQueue = new LinkedBlockingQueue<>(5000); |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* process service |
||||||
|
*/ |
||||||
|
@Autowired |
||||||
|
private ProcessService processService; |
||||||
|
|
||||||
|
|
||||||
|
@PostConstruct |
||||||
|
public void init(){ |
||||||
|
TaskWorker taskWorker = new TaskWorker(); |
||||||
|
taskWorker.start(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* put task to attemptQueue |
||||||
|
* |
||||||
|
* @param taskEvent taskEvent |
||||||
|
*/ |
||||||
|
public void putTask(TaskEvent taskEvent){ |
||||||
|
try { |
||||||
|
attemptQueue.put(taskEvent); |
||||||
|
} catch (InterruptedException e) { |
||||||
|
logger.error("put task : {} error :{}",taskEvent,e); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* task worker thread |
||||||
|
*/ |
||||||
|
class TaskWorker extends Thread { |
||||||
|
|
||||||
|
@Override |
||||||
|
public void run() { |
||||||
|
|
||||||
|
while (Stopper.isRunning()){ |
||||||
|
try { |
||||||
|
if (attemptQueue.size() == 0){ |
||||||
|
Thread.sleep(Constants.SLEEP_TIME_MILLIS); |
||||||
|
continue; |
||||||
|
} |
||||||
|
TaskEvent taskEvent = attemptQueue.take(); |
||||||
|
|
||||||
|
persist(taskEvent); |
||||||
|
|
||||||
|
}catch (Exception e){ |
||||||
|
logger.error("persist task error",e); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* persist taskEvent |
||||||
|
* @param taskEvent taskEvent |
||||||
|
*/ |
||||||
|
private void persist(TaskEvent taskEvent){ |
||||||
|
if (TaskEvent.ACK.equals(taskEvent.getType())){ |
||||||
|
processService.changeTaskState(taskEvent.getState(), |
||||||
|
taskEvent.getStartTime(), |
||||||
|
taskEvent.getWorkerAddress(), |
||||||
|
taskEvent.getExecutePath(), |
||||||
|
taskEvent.getLogPath(), |
||||||
|
taskEvent.getTaskInstanceId()); |
||||||
|
}else if (TaskEvent.RESPONSE.equals(taskEvent.getType())){ |
||||||
|
processService.changeTaskState(taskEvent.getState(), |
||||||
|
taskEvent.getEndTime(), |
||||||
|
taskEvent.getProcessId(), |
||||||
|
taskEvent.getAppIds(), |
||||||
|
taskEvent.getTaskInstanceId()); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue