Browse Source
* fix 9584 * master recall * fix ut * update logger * update delay queue * fix ut * remove sleep Co-authored-by: 进勇 <lijinyong@cai-inc.com> Co-authored-by: JinyLeeChina <jiny.li@foxmail.com>3.1.0-release
JinYong Li
2 years ago
committed by
GitHub
32 changed files with 554 additions and 81 deletions
@ -0,0 +1,62 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.master.processor; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.remote.command.Command; |
||||||
|
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskRecallCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||||
|
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; |
||||||
|
import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
import com.google.common.base.Preconditions; |
||||||
|
|
||||||
|
import io.netty.channel.Channel; |
||||||
|
|
||||||
|
/** |
||||||
|
* task recall processor |
||||||
|
*/ |
||||||
|
@Component |
||||||
|
public class TaskRecallProcessor implements NettyRequestProcessor { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(TaskRecallProcessor.class); |
||||||
|
|
||||||
|
@Autowired |
||||||
|
private TaskEventService taskEventService; |
||||||
|
|
||||||
|
/** |
||||||
|
* task ack process |
||||||
|
* |
||||||
|
* @param channel channel channel |
||||||
|
* @param command command TaskExecuteAckCommand |
||||||
|
*/ |
||||||
|
@Override |
||||||
|
public void process(Channel channel, Command command) { |
||||||
|
Preconditions.checkArgument(CommandType.TASK_RECALL == command.getType(), String.format("invalid command type : %s", command.getType())); |
||||||
|
TaskRecallCommand recallCommand = JSONUtils.parseObject(command.getBody(), TaskRecallCommand.class); |
||||||
|
logger.info("taskRecallCommand : {}", recallCommand); |
||||||
|
TaskEvent taskEvent = TaskEvent.newRecall(recallCommand, channel); |
||||||
|
taskEventService.addEvent(taskEvent); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,74 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.remote.command; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
|
||||||
|
import java.io.Serializable; |
||||||
|
|
||||||
|
/** |
||||||
|
* task recall ack command |
||||||
|
*/ |
||||||
|
public class TaskRecallAckCommand implements Serializable { |
||||||
|
|
||||||
|
private int taskInstanceId; |
||||||
|
private int status; |
||||||
|
|
||||||
|
public TaskRecallAckCommand() { |
||||||
|
super(); |
||||||
|
} |
||||||
|
|
||||||
|
public TaskRecallAckCommand(int status, int taskInstanceId) { |
||||||
|
this.status = status; |
||||||
|
this.taskInstanceId = taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public int getTaskInstanceId() { |
||||||
|
return taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTaskInstanceId(int taskInstanceId) { |
||||||
|
this.taskInstanceId = taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public int getStatus() { |
||||||
|
return status; |
||||||
|
} |
||||||
|
|
||||||
|
public void setStatus(int status) { |
||||||
|
this.status = status; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* package response command |
||||||
|
* |
||||||
|
* @return command |
||||||
|
*/ |
||||||
|
public Command convert2Command() { |
||||||
|
Command command = new Command(); |
||||||
|
command.setType(CommandType.TASK_RECALL_ACK); |
||||||
|
byte[] body = JSONUtils.toJsonByteArray(this); |
||||||
|
command.setBody(body); |
||||||
|
return command; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "TaskRecallAckCommand{" + "taskInstanceId=" + taskInstanceId + ", status=" + status + '}'; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,90 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.remote.command; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.enums.Event; |
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
|
||||||
|
import java.io.Serializable; |
||||||
|
|
||||||
|
/** |
||||||
|
* kill task recall command |
||||||
|
*/ |
||||||
|
public class TaskRecallCommand implements Serializable { |
||||||
|
|
||||||
|
/** |
||||||
|
* taskInstanceId |
||||||
|
*/ |
||||||
|
private int taskInstanceId; |
||||||
|
|
||||||
|
/** |
||||||
|
* host |
||||||
|
*/ |
||||||
|
private String host; |
||||||
|
|
||||||
|
/** |
||||||
|
* process instance id |
||||||
|
*/ |
||||||
|
private int processInstanceId; |
||||||
|
|
||||||
|
public int getTaskInstanceId() { |
||||||
|
return taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setTaskInstanceId(int taskInstanceId) { |
||||||
|
this.taskInstanceId = taskInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public String getHost() { |
||||||
|
return host; |
||||||
|
} |
||||||
|
|
||||||
|
public void setHost(String host) { |
||||||
|
this.host = host; |
||||||
|
} |
||||||
|
|
||||||
|
public int getProcessInstanceId() { |
||||||
|
return processInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
public void setProcessInstanceId(int processInstanceId) { |
||||||
|
this.processInstanceId = processInstanceId; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* package request command |
||||||
|
* |
||||||
|
* @return command |
||||||
|
*/ |
||||||
|
public Command convert2Command() { |
||||||
|
Command command = new Command(); |
||||||
|
command.setType(CommandType.TASK_RECALL); |
||||||
|
byte[] body = JSONUtils.toJsonByteArray(this); |
||||||
|
command.setBody(body); |
||||||
|
return command; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String toString() { |
||||||
|
return "TaskRecallCommand{" |
||||||
|
+ "taskInstanceId=" + taskInstanceId |
||||||
|
+ ", host='" + host + '\'' |
||||||
|
+ ", processInstanceId=" + processInstanceId |
||||||
|
+ '}'; |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,58 @@ |
|||||||
|
/* |
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||||
|
* contributor license agreements. See the NOTICE file distributed with |
||||||
|
* this work for additional information regarding copyright ownership. |
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||||
|
* (the "License"); you may not use this file except in compliance with |
||||||
|
* the License. You may obtain a copy of the License at |
||||||
|
* |
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* |
||||||
|
* Unless required by applicable law or agreed to in writing, software |
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
* See the License for the specific language governing permissions and |
||||||
|
* limitations under the License. |
||||||
|
*/ |
||||||
|
|
||||||
|
package org.apache.dolphinscheduler.server.worker.processor; |
||||||
|
|
||||||
|
import org.apache.dolphinscheduler.common.utils.JSONUtils; |
||||||
|
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus; |
||||||
|
import org.apache.dolphinscheduler.remote.command.Command; |
||||||
|
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||||
|
import org.apache.dolphinscheduler.remote.command.TaskRecallAckCommand; |
||||||
|
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||||
|
import org.apache.dolphinscheduler.server.worker.cache.ResponseCache; |
||||||
|
|
||||||
|
import org.slf4j.Logger; |
||||||
|
import org.slf4j.LoggerFactory; |
||||||
|
import org.springframework.stereotype.Component; |
||||||
|
|
||||||
|
import com.google.common.base.Preconditions; |
||||||
|
|
||||||
|
import io.netty.channel.Channel; |
||||||
|
|
||||||
|
@Component |
||||||
|
public class TaskRecallAckProcessor implements NettyRequestProcessor { |
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(TaskRecallAckProcessor.class); |
||||||
|
|
||||||
|
@Override |
||||||
|
public void process(Channel channel, Command command) { |
||||||
|
Preconditions.checkArgument(CommandType.TASK_RECALL_ACK == command.getType(), |
||||||
|
String.format("invalid command type : %s", command.getType())); |
||||||
|
|
||||||
|
TaskRecallAckCommand taskRecallAckCommand = JSONUtils.parseObject(command.getBody(), TaskRecallAckCommand.class); |
||||||
|
if (taskRecallAckCommand == null) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
if (taskRecallAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()) { |
||||||
|
ResponseCache.get().removeRecallCache(taskRecallAckCommand.getTaskInstanceId()); |
||||||
|
logger.debug("removeRecallCache: task instance id:{}", taskRecallAckCommand.getTaskInstanceId()); |
||||||
|
TaskCallbackService.remove(taskRecallAckCommand.getTaskInstanceId()); |
||||||
|
logger.debug("remove REMOTE_CHANNELS, task instance id:{}", taskRecallAckCommand.getTaskInstanceId()); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue