Browse Source
* 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * add license * modify javadoc error * TaskExecutionContext create modify * buildAckCommand taskInstanceId not set modify * java doc error modify * add comment * ExecutorManager interface add generic type * add TaskInstanceCacheManager receive Worker report result * TaskInstance setExecutePath * add TaskInstanceCacheManager to receive Worker Task result report * TaskInstanceCacheManager add remove method * add license * add dispatcht task method * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * 1，worker TaskPros use TaskExecutionContext replase 2，Master kill Task , KillTaskProcessor modify * worker remove db * ShellTask modify * master persistence processId and appIds * master persistence processId and appIds * master add kill task logic * master add kill task logic * master add kill task logic * javadoc error modifypull/2/head
qiaozhanwei
4 years ago
committed by
GitHub
53 changed files with 610 additions and 224 deletions
@ -1 +1 @@ |
|||||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
/**
* taskInstanceId
*/
private int taskInstanceId;
/**
* processId
*/
private int processId;
/**
* host
*/
private String host;
/**
InstanceId
*/
private String tenantCode;
/**
* logPath
*/
private String logPath;
/**
* executePath
*/
InstanceId
/**
InstanceId
* processId
InstanceId
private int processId;
}
InstanceId
private String host;
this.logPath = logPath;
}
public int getTaskInstanceId() {
*/
*/
}
*/
private int taskInstanceId;
this.taskInstanceId = taskInstanceId;
}
*/
/**
*/
* processId
}
*/
private int processId;
*/
* host
}
*/
private String host;
private int taskInstanceId;
}
public void setHost(String host) {
this.host = host;
}
public String getTenantCode() {
return tenantCode;
InstanceId
* host
private int taskInstanceId;
/**
this.tenantCode = tenantCode;
}
private int taskInstanceId;
private int processId;
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
InstanceId
*/
private int taskInstanceId;
*/
/**
* processId
private int processId;
command.setBody(body);
return command;
}
} |
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
/**
* task /**
InstanceId
/**
*/
/**
private int taskInstanceId;
/**
/**
/**
/**
* processId
/**
private int processId;
/**
/**
* host
/**
private String host;
}
/**
* processId
/**
private int processId;
/**
* processId
InstanceId
/**
private String host;
}
/**
* processId
*/
* processId
private int taskInstanceId;
* processId
* processId
/**
/**
*/
public Command convert2Command(){
* processId
private int processId;
* processId
* host
* processId
private String host;
private int processId;
private int processId;
InstanceId
/**
private int processId;
/**
private int processId;
*/
private int processId;
private int taskInstanceId;
private int processId;
private int processId;
/**
private int processId;
* processId
private int processId;
private int processId;
>
} |
||||||
@ -0,0 +1,116 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.entity; |
||||||
|
|
||||||
|
import java.io.Serializable; |
||||||
|
|
||||||
|
/** |
||||||
|
* master/worker task transport |
||||||
|
*/ |
||||||
|
public class DataxTaskExecutionContext implements Serializable{ |
||||||
|
|
||||||
|
/** |
||||||
|
* dataSourceId |
||||||
|
*/ |
||||||
|
private int dataSourceId; |
||||||
|
|
||||||
|
/** |
||||||
|
* sourcetype |
||||||
|
*/ |
||||||
|
private int sourcetype; |
||||||
|
|
||||||
|
/** |
||||||
|
* sourceConnectionParams |
||||||
|
*/ |
||||||
|
private String sourceConnectionParams; |
||||||
|
|
||||||
|
/** |
||||||
|
* dataTargetId |
||||||
|
*/ |
||||||
|
private int dataTargetId; |
||||||
|
|
||||||
|
/** |
||||||
|
* targetType |
||||||
|
*/ |
||||||
|
private int targetType; |
||||||
|
|
||||||
|
/** |
||||||
|
* targetConnectionParams |
||||||
|
*/ |
||||||
|
private String targetConnectionParams; |
||||||
|
|
||||||
|
public int getDataSourceId() { |
||||||
|
return dataSourceId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDataSourceId(int dataSourceId) { |
||||||
|
this.dataSourceId = dataSourceId; |
||||||
|
} |
||||||
|
|
||||||
|
public int getSourcetype() { |
||||||
|
return sourcetype; |
||||||
|
} |
||||||
|
|
||||||
|
public void setSourcetype(int sourcetype) { |
||||||
|
this.sourcetype = sourcetype; |
||||||
|
} |
||||||
|
|
||||||
|
public String getSourceConnectionParams() { |
||||||
|
return sourceConnectionParams; |
||||||
|
} |
||||||
|
|
||||||
|
public void setSourceConnectionParams(String sourceConnectionParams) { |
||||||
|
this.sourceConnectionParams = sourceConnectionParams; |
||||||
|
} |
||||||
|
|
||||||
|
public int getDataTargetId() { |
||||||
|
return dataTargetId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setDataTargetId(int dataTargetId) { |
||||||
|
this.dataTargetId = dataTargetId; |
||||||
|
} |
||||||
|
|
||||||
|
public int getTargetType() { |
||||||
|
return targetType; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTargetType(int targetType) { |
||||||
|
this.targetType = targetType; |
||||||
|
} |
||||||
|
|
||||||
|
public String getTargetConnectionParams() { |
||||||
|
return targetConnectionParams; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTargetConnectionParams(String targetConnectionParams) { |
||||||
|
this.targetConnectionParams = targetConnectionParams; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "DataxTaskExecutionContext{" + |
||||||
|
"dataSourceId=" + dataSourceId + |
||||||
|
", sourcetype=" + sourcetype + |
||||||
|
", sourceConnectionParams='" + sourceConnectionParams + '\'' + |
||||||
|
", dataTargetId=" + dataTargetId + |
||||||
|
", targetType=" + targetType + |
||||||
|
", targetConnectionParams='" + targetConnectionParams + '\'' + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,64 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.entity; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.dao.entity.UdfFunc; |
||||||
|
|
||||||
|
import java.io.Serializable; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
/** |
||||||
|
* SQL Task ExecutionContext |
||||||
|
*/ |
||||||
|
public class SQLTaskExecutionContext implements Serializable { |
||||||
|
|
||||||
|
|
||||||
|
/** |
||||||
|
* warningGroupId |
||||||
|
*/ |
||||||
|
private int warningGroupId; |
||||||
|
/** |
||||||
|
* udf function list |
||||||
|
*/ |
||||||
|
private List<UdfFunc> udfFuncList; |
||||||
|
|
||||||
|
|
||||||
|
public int getWarningGroupId() { |
||||||
|
return warningGroupId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setWarningGroupId(int warningGroupId) { |
||||||
|
this.warningGroupId = warningGroupId; |
||||||
|
} |
||||||
|
|
||||||
|
public List<UdfFunc> getUdfFuncList() { |
||||||
|
return udfFuncList; |
||||||
|
} |
||||||
|
|
||||||
|
public void setUdfFuncList(List<UdfFunc> udfFuncList) { |
||||||
|
this.udfFuncList = udfFuncList; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "SQLTaskExecutionContext{" + |
||||||
|
"warningGroupId=" + warningGroupId + |
||||||
|
", udfFuncList=" + udfFuncList + |
||||||
|
'}'; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,119 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.dispatch.executor; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.remote.NettyRemotingClient; |
||||||
|
import org.apache.dolphinscheduler.remote.command.Command; |
||||||
|
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||||
|
import org.apache.dolphinscheduler.remote.command.KillTaskRequestCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.config.NettyClientConfig; |
||||||
|
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||||
|
import org.apache.dolphinscheduler.remote.utils.Host; |
||||||
|
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; |
||||||
|
import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; |
||||||
|
import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.stereotype.Service; |
||||||
|
|
||||||
|
import java.util.Collection; |
||||||
|
import java.util.Collections; |
||||||
|
import java.util.HashSet; |
||||||
|
import java.util.Set; |
||||||
|
|
||||||
|
/** |
||||||
|
* netty executor manager |
||||||
|
*/ |
||||||
|
@Service |
||||||
|
public class NettyKillManager extends AbstractExecutorManager<Boolean>{ |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(NettyKillManager.class); |
||||||
|
/** |
||||||
|
* netty remote client |
||||||
|
*/ |
||||||
|
private final NettyRemotingClient nettyRemotingClient; |
||||||
|
|
||||||
|
public NettyKillManager(){ |
||||||
|
final NettyClientConfig clientConfig = new NettyClientConfig(); |
||||||
|
this.nettyRemotingClient = new NettyRemotingClient(clientConfig); |
||||||
|
/** |
||||||
|
* register KILL_TASK_RESPONSE command type TaskKillResponseProcessor |
||||||
|
*/ |
||||||
|
this.nettyRemotingClient.registerProcessor(CommandType.KILL_TASK_RESPONSE, new TaskKillResponseProcessor()); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* execute logic |
||||||
|
* |
||||||
|
* @param context context |
||||||
|
* @return result |
||||||
|
* @throws ExecuteException |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public Boolean execute(ExecutionContext context) throws ExecuteException { |
||||||
|
Host host = context.getHost(); |
||||||
|
Command command = buildCommand(context); |
||||||
|
try { |
||||||
|
doExecute(host, command); |
||||||
|
return true; |
||||||
|
}catch (ExecuteException ex) { |
||||||
|
logger.error(String.format("execute context : %s error", context.getContext()), ex); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
private Command buildCommand(ExecutionContext context) { |
||||||
|
KillTaskRequestCommand requestCommand = new KillTaskRequestCommand(); |
||||||
|
TaskExecutionContext taskExecutionContext = context.getContext(); |
||||||
|
|
||||||
|
requestCommand.setTaskExecutionContext(FastJsonSerializer.serializeToString(taskExecutionContext)); |
||||||
|
return requestCommand.convert2Command(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* execute logic |
||||||
|
* @param host host |
||||||
|
* @param command command |
||||||
|
* @throws ExecuteException |
||||||
|
*/ |
||||||
|
private void doExecute(final Host host, final Command command) throws ExecuteException { |
||||||
|
/** |
||||||
|
* retry count，default retry 3 |
||||||
|
*/ |
||||||
|
int retryCount = 3; |
||||||
|
boolean success = false; |
||||||
|
do { |
||||||
|
try { |
||||||
|
nettyRemotingClient.send(host, command); |
||||||
|
success = true; |
||||||
|
} catch (Exception ex) { |
||||||
|
logger.error(String.format("send command : %s to %s error", command, host), ex); |
||||||
|
retryCount--; |
||||||
|
try { |
||||||
|
Thread.sleep(100); |
||||||
|
} catch (InterruptedException ignore) {} |
||||||
|
} |
||||||
|
} while (retryCount >= 0 && !success); |
||||||
|
|
||||||
|
if (!success) { |
||||||
|
throw new ExecuteException(String.format("send command : %s to %s error", command, host)); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,61 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.processor; |
||||||
|
|
||||||
|
import io.netty.channel.Channel; |
||||||
|
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.common.utils.Preconditions; |
||||||
|
import org.apache.dolphinscheduler.remote.command.Command; |
||||||
|
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||||
|
import org.apache.dolphinscheduler.remote.command.ExecuteTaskResponseCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.command.KillTaskResponseCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||||
|
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||||
|
import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; |
||||||
|
import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; |
||||||
|
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||||
|
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||||
|
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; |
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
|
||||||
|
/** |
||||||
|
* task response processor |
||||||
|
*/ |
||||||
|
public class TaskKillResponseProcessor implements NettyRequestProcessor { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(TaskKillResponseProcessor.class); |
||||||
|
|
||||||
|
/** |
||||||
|
* task final result response |
||||||
|
* need master process , state persistence |
||||||
|
* |
||||||
|
* @param channel channel |
||||||
|
* @param command command |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void process(Channel channel, Command command) { |
||||||
|
Preconditions.checkArgument(CommandType.KILL_TASK_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); |
||||||
|
|
||||||
|
KillTaskResponseCommand responseCommand = FastJsonSerializer.deserialize(command.getBody(), KillTaskResponseCommand.class); |
||||||
|
logger.info("received command : {}", responseCommand); |
||||||
|
logger.info("已经接受到了worker杀任务的回应"); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
Loading…
Reference in new issue