qiaozhanwei
4 years ago
committed by
GitHub
19 changed files with 705 additions and 98 deletions
@ -0,0 +1,23 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.common.enums; |
||||
|
||||
public enum Event { |
||||
ACK, |
||||
RESULT; |
||||
} |
||||
@ -1 +1,110 @@
|
||||
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command;
public enum CommandType {
/**
* remove task log request,
*/
REMOVE_TAK_LOG_REQUEST,
/**
* remove task log response
*/
REMOVE_TAK_LOG_RESPONSE,
/**
* roll view log request
*/
ROLL_VIEW_LOG_REQUEST,
/**
* roll view log response
*/
ROLL_VIEW_LOG_RESPONSE,
/**
* view whole log request
*/
VIEW_WHOLE_LOG_REQUEST,
/**
* view whole log response
*/
VIEW_WHOLE_LOG_RESPONSE,
/**
* get log bytes request
*/
GET_LOG_BYTES_REQUEST,
/**
* get log bytes response
*/
GET_LOG_BYTES_RESPONSE,
WORKER_REQUEST,
MASTER_RESPONSE,
/**
* execute task request
*/
TASK_EXECUTE_REQUEST,
/**
* execute task ack
*/
TASK_EXECUTE_ACK,
/**
* execute task response
*/
TASK_EXECUTE_RESPONSE,
/**
* kill task
*/
TASK_KILL_REQUEST,
/**
* kill task response
*/
TASK_KILL_RESPONSE,
/**
* ping
*/
PING,
/**
* pong
*/
PONG;
} |
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.remote.command; |
||||
|
||||
public enum CommandType { |
||||
|
||||
/** |
||||
* remove task log request, |
||||
*/ |
||||
REMOVE_TAK_LOG_REQUEST, |
||||
|
||||
/** |
||||
* remove task log response |
||||
*/ |
||||
REMOVE_TAK_LOG_RESPONSE, |
||||
|
||||
/** |
||||
* roll view log request |
||||
*/ |
||||
ROLL_VIEW_LOG_REQUEST, |
||||
|
||||
/** |
||||
* roll view log response |
||||
*/ |
||||
ROLL_VIEW_LOG_RESPONSE, |
||||
|
||||
/** |
||||
* view whole log request |
||||
*/ |
||||
VIEW_WHOLE_LOG_REQUEST, |
||||
|
||||
/** |
||||
* view whole log response |
||||
*/ |
||||
VIEW_WHOLE_LOG_RESPONSE, |
||||
|
||||
/** |
||||
* get log bytes request |
||||
*/ |
||||
GET_LOG_BYTES_REQUEST, |
||||
|
||||
/** |
||||
* get log bytes response |
||||
*/ |
||||
GET_LOG_BYTES_RESPONSE, |
||||
|
||||
|
||||
WORKER_REQUEST, |
||||
MASTER_RESPONSE, |
||||
|
||||
/** |
||||
* execute task request |
||||
*/ |
||||
TASK_EXECUTE_REQUEST, |
||||
|
||||
/** |
||||
* execute task ack |
||||
*/ |
||||
TASK_EXECUTE_ACK, |
||||
|
||||
/** |
||||
* execute task response |
||||
*/ |
||||
TASK_EXECUTE_RESPONSE, |
||||
|
||||
/** |
||||
* db task ack |
||||
*/ |
||||
DB_TASK_ACK, |
||||
|
||||
/** |
||||
* db task response |
||||
*/ |
||||
DB_TASK_RESPONSE, |
||||
|
||||
/** |
||||
* kill task |
||||
*/ |
||||
TASK_KILL_REQUEST, |
||||
|
||||
/** |
||||
* kill task response |
||||
*/ |
||||
TASK_KILL_RESPONSE, |
||||
|
||||
/** |
||||
* ping |
||||
*/ |
||||
PING, |
||||
|
||||
/** |
||||
* pong |
||||
*/ |
||||
PONG; |
||||
} |
||||
|
||||
@ -0,0 +1,72 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.remote.command; |
||||
|
||||
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||
|
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* db task ack request command |
||||
*/ |
||||
public class DBTaskAckCommand implements Serializable { |
||||
|
||||
private int taskInstanceId; |
||||
private int status; |
||||
|
||||
public DBTaskAckCommand(int status,int taskInstanceId) { |
||||
this.status = status; |
||||
this.taskInstanceId = taskInstanceId; |
||||
} |
||||
|
||||
public int getTaskInstanceId() { |
||||
return taskInstanceId; |
||||
} |
||||
|
||||
public void setTaskInstanceId(int taskInstanceId) { |
||||
this.taskInstanceId = taskInstanceId; |
||||
} |
||||
|
||||
public int getStatus() { |
||||
return status; |
||||
} |
||||
|
||||
public void setStatus(int status) { |
||||
this.status = status; |
||||
} |
||||
|
||||
/** |
||||
* package response command |
||||
* @return command |
||||
*/ |
||||
public Command convert2Command(){ |
||||
Command command = new Command(); |
||||
command.setType(CommandType.DB_TASK_ACK); |
||||
byte[] body = FastJsonSerializer.serialize(this); |
||||
command.setBody(body); |
||||
return command; |
||||
} |
||||
|
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "DBTaskAckCommand{" + |
||||
"taskInstanceId=" + taskInstanceId + |
||||
", status=" + status + |
||||
'}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,71 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
package org.apache.dolphinscheduler.remote.command; |
||||
|
||||
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||
|
||||
import java.io.Serializable; |
||||
|
||||
/** |
||||
* db task final result response command |
||||
*/ |
||||
public class DBTaskResponseCommand implements Serializable { |
||||
|
||||
private int taskInstanceId; |
||||
private int status; |
||||
|
||||
public DBTaskResponseCommand(int status,int taskInstanceId) { |
||||
this.status = status; |
||||
this.taskInstanceId = taskInstanceId; |
||||
} |
||||
|
||||
public int getStatus() { |
||||
return status; |
||||
} |
||||
|
||||
public void setStatus(int status) { |
||||
this.status = status; |
||||
} |
||||
|
||||
public int getTaskInstanceId() { |
||||
return taskInstanceId; |
||||
} |
||||
|
||||
public void setTaskInstanceId(int taskInstanceId) { |
||||
this.taskInstanceId = taskInstanceId; |
||||
} |
||||
|
||||
/** |
||||
* package response command |
||||
* @return command |
||||
*/ |
||||
public Command convert2Command(){ |
||||
Command command = new Command(); |
||||
command.setType(CommandType.DB_TASK_RESPONSE); |
||||
byte[] body = FastJsonSerializer.serialize(this); |
||||
command.setBody(body); |
||||
return command; |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "DBTaskResponseCommand{" + |
||||
"taskInstanceId=" + taskInstanceId + |
||||
", status=" + status + |
||||
'}'; |
||||
} |
||||
} |
||||
@ -0,0 +1,94 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.cache; |
||||
|
||||
import org.apache.dolphinscheduler.common.enums.Event; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
|
||||
import java.util.Map; |
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
|
||||
/** |
||||
* Responce Cache : cache worker send master result |
||||
*/ |
||||
public class ResponceCache { |
||||
|
||||
private static final ResponceCache instance = new ResponceCache(); |
||||
|
||||
private ResponceCache(){} |
||||
|
||||
public static ResponceCache get(){ |
||||
return instance; |
||||
} |
||||
|
||||
private Map<Integer,Command> ackCache = new ConcurrentHashMap<>(); |
||||
private Map<Integer,Command> responseCache = new ConcurrentHashMap<>(); |
||||
|
||||
|
||||
/** |
||||
* cache response |
||||
* @param taskInstanceId taskInstanceId |
||||
* @param command command |
||||
* @param event event ACK/RESULT |
||||
*/ |
||||
public void cache(Integer taskInstanceId, Command command, Event event){ |
||||
switch (event){ |
||||
case ACK: |
||||
ackCache.put(taskInstanceId,command); |
||||
break; |
||||
case RESULT: |
||||
responseCache.put(taskInstanceId,command); |
||||
break; |
||||
default: |
||||
throw new IllegalArgumentException("invalid event type : " + event); |
||||
} |
||||
} |
||||
|
||||
|
||||
/** |
||||
* remove ack cache |
||||
* @param taskInstanceId taskInstanceId |
||||
*/ |
||||
public void removeAckCache(Integer taskInstanceId){ |
||||
ackCache.remove(taskInstanceId); |
||||
} |
||||
|
||||
/** |
||||
* remove reponse cache |
||||
* @param taskInstanceId taskInstanceId |
||||
*/ |
||||
public void removeResponseCache(Integer taskInstanceId){ |
||||
responseCache.remove(taskInstanceId); |
||||
} |
||||
|
||||
/** |
||||
* getAckCache |
||||
* @return getAckCache |
||||
*/ |
||||
public Map<Integer,Command> getAckCache(){ |
||||
return ackCache; |
||||
} |
||||
|
||||
/** |
||||
* getResponseCache |
||||
* @return getResponseCache |
||||
*/ |
||||
public Map<Integer,Command> getResponseCache(){ |
||||
return responseCache; |
||||
} |
||||
} |
||||
@ -0,0 +1,56 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.processor; |
||||
|
||||
import io.netty.channel.Channel; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.utils.Preconditions; |
||||
import org.apache.dolphinscheduler.remote.command.*; |
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||
import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* db task ack processor |
||||
*/ |
||||
public class DBTaskAckProcessor implements NettyRequestProcessor { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(DBTaskAckProcessor.class); |
||||
|
||||
|
||||
@Override |
||||
public void process(Channel channel, Command command) { |
||||
Preconditions.checkArgument(CommandType.DB_TASK_ACK == command.getType(), |
||||
String.format("invalid command type : %s", command.getType())); |
||||
|
||||
DBTaskAckCommand taskAckCommand = FastJsonSerializer.deserialize( |
||||
command.getBody(), DBTaskAckCommand.class); |
||||
|
||||
if (taskAckCommand == null){ |
||||
return; |
||||
} |
||||
|
||||
if (taskAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()){ |
||||
ResponceCache.get().removeAckCache(taskAckCommand.getTaskInstanceId()); |
||||
} |
||||
} |
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,58 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.processor; |
||||
|
||||
import io.netty.channel.Channel; |
||||
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; |
||||
import org.apache.dolphinscheduler.common.utils.Preconditions; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.remote.command.CommandType; |
||||
import org.apache.dolphinscheduler.remote.command.DBTaskResponseCommand; |
||||
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; |
||||
import org.apache.dolphinscheduler.remote.utils.FastJsonSerializer; |
||||
import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
|
||||
/** |
||||
* db task response processor |
||||
*/ |
||||
public class DBTaskResponseProcessor implements NettyRequestProcessor { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(DBTaskResponseProcessor.class); |
||||
|
||||
|
||||
@Override |
||||
public void process(Channel channel, Command command) { |
||||
Preconditions.checkArgument(CommandType.DB_TASK_RESPONSE == command.getType(), |
||||
String.format("invalid command type : %s", command.getType())); |
||||
|
||||
DBTaskResponseCommand taskResponseCommand = FastJsonSerializer.deserialize( |
||||
command.getBody(), DBTaskResponseCommand.class); |
||||
|
||||
if (taskResponseCommand == null){ |
||||
return; |
||||
} |
||||
|
||||
if (taskResponseCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()){ |
||||
ResponceCache.get().removeResponseCache(taskResponseCommand.getTaskInstanceId()); |
||||
} |
||||
} |
||||
|
||||
|
||||
} |
||||
@ -0,0 +1,92 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.server.worker.runner; |
||||
|
||||
import org.apache.dolphinscheduler.common.thread.Stopper; |
||||
|
||||
import org.apache.dolphinscheduler.common.thread.ThreadUtils; |
||||
import org.apache.dolphinscheduler.remote.command.Command; |
||||
import org.apache.dolphinscheduler.server.worker.cache.ResponceCache; |
||||
import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; |
||||
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; |
||||
import org.slf4j.Logger; |
||||
import org.slf4j.LoggerFactory; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
import java.util.Map; |
||||
|
||||
/** |
||||
* Retry Report Task Status Thread |
||||
*/ |
||||
@Component |
||||
public class RetryReportTaskStatusThread implements Runnable { |
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(RetryReportTaskStatusThread.class); |
||||
|
||||
/** |
||||
* every 5 minutes |
||||
*/ |
||||
private static long RETRY_REPORT_TASK_STATUS_TIME = 5 * 60 * 1000L; |
||||
/** |
||||
* task callback service |
||||
*/ |
||||
private final TaskCallbackService taskCallbackService; |
||||
|
||||
public void start(){ |
||||
Thread thread = new Thread(this,"RetryReportTaskStatusThread"); |
||||
thread.start(); |
||||
} |
||||
|
||||
public RetryReportTaskStatusThread(){ |
||||
this.taskCallbackService = SpringApplicationContext.getBean(TaskCallbackService.class); |
||||
} |
||||
|
||||
/** |
||||
* retry ack/response |
||||
*/ |
||||
@Override |
||||
public void run() { |
||||
ResponceCache responceCache = ResponceCache.get(); |
||||
|
||||
while (Stopper.isRunning()){ |
||||
try { |
||||
if (!responceCache.getAckCache().isEmpty()){ |
||||
Map<Integer,Command> ackCache = responceCache.getAckCache(); |
||||
for (Map.Entry<Integer, Command> entry : ackCache.entrySet()){ |
||||
Integer taskInstanceId = entry.getKey(); |
||||
Command ackCommand = entry.getValue(); |
||||
taskCallbackService.sendAck(taskInstanceId,ackCommand); |
||||
} |
||||
} |
||||
|
||||
if (!responceCache.getResponseCache().isEmpty()){ |
||||
Map<Integer,Command> responseCache = responceCache.getResponseCache(); |
||||
for (Map.Entry<Integer, Command> entry : responseCache.entrySet()){ |
||||
Integer taskInstanceId = entry.getKey(); |
||||
Command responseCommand = entry.getValue(); |
||||
taskCallbackService.sendAck(taskInstanceId,responseCommand); |
||||
} |
||||
} |
||||
}catch (Exception e){ |
||||
logger.warn("retry report task status error", e); |
||||
} |
||||
|
||||
ThreadUtils.sleep(RETRY_REPORT_TASK_STATUS_TIME); |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue