Browse Source
* 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * 1, master persistent task 2. extract master and worker communication model * add license * modify javadoc error * TaskExecutionContext create modify * buildAckCommand taskInstanceId not set modify * java doc error modify * add comment * ExecutorManager interface add generic type * add TaskInstanceCacheManager receive Worker report result * TaskInstance setExecutePath * add TaskInstanceCacheManager to receive Worker Task result report * TaskInstanceCacheManager add remove method * add license * add dispatcht task method * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * AbstractCommandExecutor remove db access * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * taskInstanceCache is null ,need load from db * 1，worker TaskPros use TaskExecutionContext replase 2，Master kill Task , KillTaskProcessor modify * worker remove db * ShellTask modify * master persistence processId and appIds * master persistence processId and appIds * master add kill task logic * master add kill task logic * master add kill task logic * javadoc error modifypull/2/head
qiaozhanwei
5 years ago
committed by
GitHub
53 changed files with 610 additions and 224 deletions
@ -1 +1 @@
|
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
/**
* taskInstanceId
*/
private int taskInstanceId;
/**
* processId
*/
private int processId;
/**
* host
*/
private String host;
/**
InstanceId
*/
private String tenantCode;
/**
* logPath
*/
private String logPath;
/**
* executePath
*/
InstanceId
/**
InstanceId
* processId
InstanceId
private int processId;
}
InstanceId
private String host;
this.logPath = logPath;
}
public int getTaskInstanceId() {
*/
*/
}
*/
private int taskInstanceId;
this.taskInstanceId = taskInstanceId;
}
*/
/**
*/
* processId
}
*/
private int processId;
*/
* host
}
*/
private String host;
private int taskInstanceId;
}
public void setHost(String host) {
this.host = host;
}
public String getTenantCode() {
return tenantCode;
InstanceId
* host
private int taskInstanceId;
/**
this.tenantCode = tenantCode;
}
private int taskInstanceId;
private int processId;
return executePath;
}
public void setExecutePath(String executePath) {
this.executePath = executePath;
}
/**
InstanceId
*/
private int taskInstanceId;
*/
/**
* processId
private int processId;
command.setBody(body);
return command;
}
} |
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer;
import java.io.Serializable;
/**
* kill task request command
*/
public class KillTaskRequestCommand implements Serializable {
/**
* task /**
InstanceId
/**
*/
/**
private int taskInstanceId;
/**
/**
/**
/**
* processId
/**
private int processId;
/**
/**
* host
/**
private String host;
}
/**
* processId
/**
private int processId;
/**
* processId
InstanceId
/**
private String host;
}
/**
* processId
*/
* processId
private int taskInstanceId;
* processId
* processId
/**
/**
*/
public Command convert2Command(){
* processId
private int processId;
* processId
* host
* processId
private String host;
private int processId;
private int processId;
InstanceId
/**
private int processId;
/**
private int processId;
*/
private int processId;
private int taskInstanceId;
private int processId;
private int processId;
/**
private int processId;
* processId
private int processId;
private int processId;
>
} |
||||
@ -0,0 +1,116 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.entity; |
||||
|
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* master/worker task transport |
||||
*/ |
||||
public class DataxTaskExecutionContext implements Serializable{ |
||||
|
||||
/** |
||||
* dataSourceId |
||||
*/ |
||||
private int dataSourceId; |
||||
|
||||
/** |
||||
* sourcetype |
||||
*/ |
||||
private int sourcetype; |
||||
|
||||
/** |
||||
* sourceConnectionParams |
||||
*/ |
||||
private String sourceConnectionParams; |
||||
|
||||
/** |
||||
* dataTargetId |
||||
*/ |
||||
private int dataTargetId; |
||||
|
||||
/** |
||||
* targetType |
||||
*/ |
||||
private int targetType; |
||||
|
||||
/** |
||||
* targetConnectionParams |
||||
*/ |
||||
private String targetConnectionParams; |
||||
|
||||
public int getDataSourceId() { |
||||
return dataSourceId; |
||||
} |
||||
|
||||
public void setDataSourceId(int dataSourceId) { |
||||
this.dataSourceId = dataSourceId; |
||||
} |
||||
|
||||
public int getSourcetype() { |
||||
return sourcetype; |
||||
} |
||||
|
||||
public void setSourcetype(int sourcetype) { |
||||
this.sourcetype = sourcetype; |
||||
} |
||||
|
||||
public String getSourceConnectionParams() { |
||||
return sourceConnectionParams; |
||||
} |
||||
|
||||
public void setSourceConnectionParams(String sourceConnectionParams) { |
||||
this.sourceConnectionParams = sourceConnectionParams; |
||||
} |
||||
|
||||
public int getDataTargetId() { |
||||
return dataTargetId; |
||||
} |
||||
|
||||
public void setDataTargetId(int dataTargetId) { |
||||
this.dataTargetId = dataTargetId; |
||||
} |
||||
|
||||
public int getTargetType() { |
||||
return targetType; |
||||
} |
||||
|
||||
public void setTargetType(int targetType) { |
||||
this.targetType = targetType; |
||||
} |
||||
|
||||
public String getTargetConnectionParams() { |
||||
return targetConnectionParams; |
||||
} |
||||
|
||||
public void setTargetConnectionParams(String targetConnectionParams) { |
||||
this.targetConnectionParams = targetConnectionParams; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "DataxTaskExecutionContext{" + |
||||
"dataSourceId=" + dataSourceId + |
||||
", sourcetype=" + sourcetype + |
||||
", sourceConnectionParams='" + sourceConnectionParams + '\'' + |
||||
", dataTargetId=" + dataTargetId + |
||||
", targetType=" + targetType + |
||||
", targetConnectionParams='" + targetConnectionParams + '\'' + |
||||
'}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,64 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.entity; |
||||
|
||||
import org.apache.dolphinscheduler.dao.entity.UdfFunc; |
||||
|
||||
import java.io.Serializable; |
||||
import java.util.List; |
||||
|
||||
/** |
||||
* SQL Task ExecutionContext |
||||
*/ |
||||
public class SQLTaskExecutionContext implements Serializable { |
||||
|
||||
|
||||
/** |
||||
* warningGroupId |
||||
*/ |
||||
private int warningGroupId; |
||||
/** |
||||
* udf function list |
||||
*/ |
||||
private List<UdfFunc> udfFuncList; |
||||
|
||||
|
||||
public int getWarningGroupId() { |
||||
return warningGroupId; |
||||
} |
||||
|
||||
public void setWarningGroupId(int warningGroupId) { |
||||
this.warningGroupId = warningGroupId; |
||||
} |
||||
|
||||
public List<UdfFunc> getUdfFuncList() { |
||||
return udfFuncList; |
||||
} |
||||
|
||||
public void setUdfFuncList(List<UdfFunc> udfFuncList) { |
||||
this.udfFuncList = udfFuncList; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "SQLTaskExecutionContext{" + |
||||
"warningGroupId=" + warningGroupId + |
||||
", udfFuncList=" + udfFuncList + |
||||
'}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,119 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.dispatch.executor; |
||||
|
||||
import org.apache.dolphinscheduler.remote.NettyRemotingClient; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||
import org.apache.dolphinscheduler.remote.command.KillTaskRequestCommand; |
||||
import org.apache.dolphinscheduler.remote.config.NettyClientConfig; |
||||
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||
import org.apache.dolphinscheduler.remote.utils.Host; |
||||
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; |
||||
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; |
||||
import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; |
||||
import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.stereotype.Service; |
||||
|
||||
import java.util.Collection; |
||||
import java.util.Collections; |
||||
import java.util.HashSet; |
||||
import java.util.Set; |
||||
|
||||
/** |
||||
* netty executor manager |
||||
*/ |
||||
@Service |
||||
public class NettyKillManager extends AbstractExecutorManager<Boolean>{ |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(NettyKillManager.class); |
||||
/** |
||||
* netty remote client |
||||
*/ |
||||
private final NettyRemotingClient nettyRemotingClient; |
||||
|
||||
public NettyKillManager(){ |
||||
final NettyClientConfig clientConfig = new NettyClientConfig(); |
||||
this.nettyRemotingClient = new NettyRemotingClient(clientConfig); |
||||
/** |
||||
* register KILL_TASK_RESPONSE command type TaskKillResponseProcessor |
||||
*/ |
||||
this.nettyRemotingClient.registerProcessor(CommandType.KILL_TASK_RESPONSE, new TaskKillResponseProcessor()); |
||||
} |
||||
|
||||
/** |
||||
* execute logic |
||||
* |
||||
* @param context context |
||||
* @return result |
||||
* @throws ExecuteException |
||||
*/ |
||||
@Override |
||||
public Boolean execute(ExecutionContext context) throws ExecuteException { |
||||
Host host = context.getHost(); |
||||
Command command = buildCommand(context); |
||||
try { |
||||
doExecute(host, command); |
||||
return true; |
||||
}catch (ExecuteException ex) { |
||||
logger.error(String.format("execute context : %s error", context.getContext()), ex); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
|
||||
private Command buildCommand(ExecutionContext context) { |
||||
KillTaskRequestCommand requestCommand = new KillTaskRequestCommand(); |
||||
TaskExecutionContext taskExecutionContext = context.getContext(); |
||||
|
||||
requestCommand.setTaskExecutionContext(FastJsonSerializer.serializeToString(taskExecutionContext)); |
||||
return requestCommand.convert2Command(); |
||||
} |
||||
|
||||
/** |
||||
* execute logic |
||||
* @param host host |
||||
* @param command command |
||||
* @throws ExecuteException |
||||
*/ |
||||
private void doExecute(final Host host, final Command command) throws ExecuteException { |
||||
/** |
||||
* retry count，default retry 3 |
||||
*/ |
||||
int retryCount = 3; |
||||
boolean success = false; |
||||
do { |
||||
try { |
||||
nettyRemotingClient.send(host, command); |
||||
success = true; |
||||
} catch (Exception ex) { |
||||
logger.error(String.format("send command : %s to %s error", command, host), ex); |
||||
retryCount--; |
||||
try { |
||||
Thread.sleep(100); |
||||
} catch (InterruptedException ignore) {} |
||||
} |
||||
} while (retryCount >= 0 && !success); |
||||
|
||||
if (!success) { |
||||
throw new ExecuteException(String.format("send command : %s to %s error", command, host)); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,61 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.master.processor; |
||||
|
||||
import io.netty.channel.Channel; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.utils.Preconditions; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||
import org.apache.dolphinscheduler.remote.command.ExecuteTaskResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.command.KillTaskResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||
import org.apache.dolphinscheduler.server.master.cache.TaskInstanceCacheManager; |
||||
import org.apache.dolphinscheduler.server.master.cache.impl.TaskInstanceCacheManagerImpl; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.apache.dolphinscheduler.service.process.ProcessService; |
||||
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* task response processor |
||||
*/ |
||||
public class TaskKillResponseProcessor implements NettyRequestProcessor { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(TaskKillResponseProcessor.class); |
||||
|
||||
/** |
||||
* task final result response |
||||
* need master process , state persistence |
||||
* |
||||
* @param channel channel |
||||
* @param command command |
||||
*/ |
||||
@Override |
||||
public void process(Channel channel, Command command) { |
||||
Preconditions.checkArgument(CommandType.KILL_TASK_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); |
||||
|
||||
KillTaskResponseCommand responseCommand = FastJsonSerializer.deserialize(command.getBody(), KillTaskResponseCommand.class); |
||||
logger.info("received command : {}", responseCommand); |
||||
logger.info("已经接受到了worker杀任务的回应"); |
||||
} |
||||
|
||||
|
||||
} |
||||
Loading…
Reference in new issue