[FIX_#3789][remote]cherry pick from dev to support netty heart beat

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java

@@ -18,10 +18,16 @@
 package org.apache.dolphinscheduler.remote;
 
 import io.netty.bootstrap.Bootstrap;
-import io.netty.channel.*;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
+
 import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
 import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
 import org.apache.dolphinscheduler.remote.command.Command;
@@ -38,6 +44,8 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.remote.utils.CallerThreadExecutePolicy;
 import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
+import org.apache.dolphinscheduler.remote.utils.NettyUtils;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -76,7 +84,7 @@ public class NettyRemotingClient {
     /**
      * worker group
      */
-    private final NioEventLoopGroup workerGroup;
+    private final EventLoopGroup workerGroup;
 
     /**
      * client config
@@ -105,10 +113,21 @@ public class NettyRemotingClient {
 
     /**
      * client init
+     *
      * @param clientConfig client config
      */
-    public NettyRemotingClient(final NettyClientConfig clientConfig){
+    public NettyRemotingClient(final NettyClientConfig clientConfig) {
         this.clientConfig = clientConfig;
+        if (NettyUtils.useEpoll()) {
+            this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet()));
+                }
+            });
+        } else {
             this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() {
                 private AtomicInteger threadIndex = new AtomicInteger(0);
 
@@ -117,6 +136,7 @@ public class NettyRemotingClient {
                     return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet()));
                 }
             });
+        }
         this.callbackExecutor = new ThreadPoolExecutor(5, 10, 1, TimeUnit.MINUTES,
             new LinkedBlockingQueue<>(1000), new NamedThreadFactory("CallbackExecutor", 10),
             new CallerThreadExecutePolicy());
@@ -130,15 +150,16 @@ public class NettyRemotingClient {
     /**
      * start
      */
-    private void start(){
+    private void start() {
 
         this.bootstrap
             .group(this.workerGroup)
-                .channel(NioSocketChannel.class)
+            .channel(NettyUtils.getSocketChannelClass())
             .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive())
             .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay())
             .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize())
             .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize())
+            .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis())
             .handler(new ChannelInitializer<SocketChannel>() {
                 @Override
                 public void initChannel(SocketChannel ch) throws Exception {
@@ -160,6 +181,7 @@ public class NettyRemotingClient {
 
     /**
      * async send
+     *
      * @param host           host
      * @param command        command
      * @param timeoutMillis  timeoutMillis
@@ -182,7 +204,7 @@ public class NettyRemotingClient {
          *  control concurrency number
          */
         boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS);
-        if(acquired){
+        if (acquired) {
             final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore);
 
             /**
@@ -193,11 +215,11 @@ public class NettyRemotingClient {
                 invokeCallback,
                 releaseSemaphore);
             try {
-                channel.writeAndFlush(command).addListener(new ChannelFutureListener(){
+                channel.writeAndFlush(command).addListener(new ChannelFutureListener() {
 
                     @Override
                     public void operationComplete(ChannelFuture future) throws Exception {
-                        if(future.isSuccess()){
+                        if (future.isSuccess()) {
                             responseFuture.setSendOk(true);
                             return;
                         } else {
@@ -207,18 +229,18 @@ public class NettyRemotingClient {
                         responseFuture.putResponse(null);
                         try {
                             responseFuture.executeInvokeCallback();
-                        } catch (Throwable ex){
+                        } catch (Throwable ex) {
                             logger.error("execute callback error", ex);
-                        } finally{
+                        } finally {
                             responseFuture.release();
                         }
                     }
                 });
-            } catch (Throwable ex){
+            } catch (Throwable ex) {
                 responseFuture.release();
                 throw new RemotingException(String.format("send command to host: %s failed", host), ex);
             }
-        } else{
+        } else {
             String message = String.format("try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d",
                 timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits());
             throw new RemotingTooMuchRequestException(message);
@@ -227,6 +249,7 @@ public class NettyRemotingClient {
 
     /**
      * sync send
+     *
      * @param host          host
      * @param command       command
      * @param timeoutMillis timeoutMillis
@@ -244,7 +267,7 @@ public class NettyRemotingClient {
         channel.writeAndFlush(command).addListener(new ChannelFutureListener() {
             @Override
             public void operationComplete(ChannelFuture future) throws Exception {
-                if(future.isSuccess()){
+                if (future.isSuccess()) {
                     responseFuture.setSendOk(true);
                     return;
                 } else {
@@ -259,10 +282,10 @@ public class NettyRemotingClient {
          * sync wait for result
          */
         Command result = responseFuture.waitResponse();
-        if(result == null){
+        if (result == null) {
-            if(responseFuture.isSendOK()){
+            if (responseFuture.isSendOK()) {
                 throw new RemotingTimeoutException(host.toString(), timeoutMillis, responseFuture.getCause());
-            } else{
+            } else {
                 throw new RemotingException(host.toString(), responseFuture.getCause());
             }
         }
@@ -271,6 +294,7 @@ public class NettyRemotingClient {
 
     /**
      * send task
+     *
      * @param host    host
      * @param command command
      * @throws RemotingException
@@ -297,6 +321,7 @@ public class NettyRemotingClient {
 
     /**
      * register processor
+     *
      * @param commandType command type
      * @param processor   processor
      */
@@ -317,12 +342,13 @@ public class NettyRemotingClient {
 
     /**
      * get channel
+     *
      * @param host
      * @return
      */
     public Channel getChannel(Host host) {
         Channel channel = channels.get(host);
-        if(channel != null && channel.isActive()){
+        if (channel != null && channel.isActive()) {
             return channel;
         }
         return createChannel(host, true);
@@ -330,6 +356,7 @@ public class NettyRemotingClient {
 
     /**
      * create channel
+     *
      * @param host   host
      * @param isSync sync flag
      * @return channel
@@ -337,10 +364,10 @@ public class NettyRemotingClient {
     public Channel createChannel(Host host, boolean isSync) {
         ChannelFuture future;
         try {
-            synchronized (bootstrap){
+            synchronized (bootstrap) {
                 future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort()));
             }
-            if(isSync){
+            if (isSync) {
                 future.sync();
             }
             if (future.isSuccess()) {
@@ -358,16 +385,16 @@ public class NettyRemotingClient {
      * close
      */
     public void close() {
-        if(isStarted.compareAndSet(true, false)){
+        if (isStarted.compareAndSet(true, false)) {
             try {
                 closeChannels();
-                if(workerGroup != null){
+                if (workerGroup != null) {
                     this.workerGroup.shutdownGracefully();
                 }
-                if(callbackExecutor != null){
+                if (callbackExecutor != null) {
                     this.callbackExecutor.shutdownNow();
                 }
-                if(this.responseFutureExecutor != null){
+                if (this.responseFutureExecutor != null) {
                     this.responseFutureExecutor.shutdownNow();
                 }
             } catch (Exception ex) {
@@ -380,7 +407,7 @@ public class NettyRemotingClient {
     /**
      * close channels
      */
-    private void closeChannels(){
+    private void closeChannels() {
         for (Channel channel : this.channels.values()) {
             channel.close();
         }
@@ -389,11 +416,12 @@ public class NettyRemotingClient {
 
     /**
      * close channel
+     *
      * @param host host
      */
-    public void closeChannel(Host host){
+    public void closeChannel(Host host) {
         Channel channel = this.channels.remove(host);
-        if(channel != null){
+        if (channel != null) {
             channel.close();
         }
     }

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java

@@ -17,14 +17,6 @@
 
 package org.apache.dolphinscheduler.remote;
 
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelOption;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
 import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
 import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -32,8 +24,7 @@ import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
 import org.apache.dolphinscheduler.remote.handler.NettyServerHandler;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.remote.utils.Constants;
-import org.slf4j.Logger;
+import org.apache.dolphinscheduler.remote.utils.NettyUtils;
-import org.slf4j.LoggerFactory;
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -41,6 +32,19 @@ import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.epoll.EpollEventLoopGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+
 /**
  * remoting netty server
  */
@@ -66,12 +70,12 @@ public class NettyRemotingServer {
     /**
      * boss group
      */
-    private final NioEventLoopGroup bossGroup;
+    private final EventLoopGroup bossGroup;
 
     /**
      * worker group
      */
-    private final NioEventLoopGroup workGroup;
+    private final EventLoopGroup workGroup;
 
     /**
      * server config
@@ -93,9 +97,27 @@ public class NettyRemotingServer {
      *
      * @param serverConfig server config
      */
-    public NettyRemotingServer(final NettyServerConfig serverConfig){
+    public NettyRemotingServer(final NettyServerConfig serverConfig) {
         this.serverConfig = serverConfig;
+        if (NettyUtils.useEpoll()) {
+            this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
 
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet()));
+                }
+            });
+
+            this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() {
+                private AtomicInteger threadIndex = new AtomicInteger(0);
+
+                @Override
+                public Thread newThread(Runnable r) {
+                    return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet()));
+                }
+            });
+        } else {
             this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() {
                 private AtomicInteger threadIndex = new AtomicInteger(0);
 
@@ -114,29 +136,26 @@ public class NettyRemotingServer {
                 }
             });
         }
+    }
 
     /**
      * server start
      */
-    public void start(){
+    public void start() {
-
+        if (isStarted.compareAndSet(false, true)) {
-        if(this.isStarted.get()){
-            return;
-        }
-
             this.serverBootstrap
                 .group(this.bossGroup, this.workGroup)
-                .channel(NioServerSocketChannel.class)
+                .channel(NettyUtils.getServerSocketChannelClass())
                 .option(ChannelOption.SO_REUSEADDR, true)
                 .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog())
                 .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive())
                 .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay())
                 .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize())
                 .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize())
-                .childHandler(new ChannelInitializer<NioSocketChannel>() {
+                .childHandler(new ChannelInitializer<SocketChannel>() {
 
                     @Override
-                    protected void initChannel(NioSocketChannel ch) throws Exception {
+                    protected void initChannel(SocketChannel ch) throws Exception {
                         initNettyChannel(ch);
                     }
                 });
@@ -145,7 +164,7 @@ public class NettyRemotingServer {
             try {
                 future = serverBootstrap.bind(serverConfig.getListenPort()).sync();
             } catch (Exception e) {
-            logger.error("NettyRemotingServer bind fail {}, exit",e.getMessage(), e);
+                logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e);
                 throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()));
             }
             if (future.isSuccess()) {
@@ -155,16 +174,15 @@ public class NettyRemotingServer {
             } else {
                 throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()));
             }
-        //
+        }
-        isStarted.compareAndSet(false, true);
     }
 
     /**
      * init netty channel
+     *
      * @param ch socket channel
-     * @throws Exception
      */
-    private void initNettyChannel(NioSocketChannel ch) throws Exception{
+    private void initNettyChannel(SocketChannel ch) {
         ChannelPipeline pipeline = ch.pipeline();
         pipeline.addLast("encoder", encoder);
         pipeline.addLast("decoder", new NettyDecoder());
@@ -173,6 +191,7 @@ public class NettyRemotingServer {
 
     /**
      * register processor
+     *
      * @param commandType command type
      * @param processor   processor
      */
@@ -193,6 +212,7 @@ public class NettyRemotingServer {
 
     /**
      * get default thread executor
+     *
      * @return thread executor
      */
     public ExecutorService getDefaultExecutor() {
@@ -200,12 +220,12 @@ public class NettyRemotingServer {
     }
 
     public void close() {
-        if(isStarted.compareAndSet(true, false)){
+        if (isStarted.compareAndSet(true, false)) {
             try {
-                if(bossGroup != null){
+                if (bossGroup != null) {
                     this.bossGroup.shutdownGracefully();
                 }
-                if(workGroup != null){
+                if (workGroup != null) {
                     this.workGroup.shutdownGracefully();
                 }
                 if(defaultExecutor != null){

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java

@@ -98,6 +98,11 @@ public enum CommandType {
      */
     TASK_KILL_RESPONSE,
 
+    /**
+     * HEART_BEAT
+     */
+    HEART_BEAT,
+
     /**
      * ping
      */

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java

@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.remote.handler;
 
-import io.netty.channel.*;
 import org.apache.dolphinscheduler.remote.NettyRemotingClient;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -25,14 +25,22 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 import org.apache.dolphinscheduler.remote.utils.Constants;
 import org.apache.dolphinscheduler.remote.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.RejectedExecutionException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.timeout.IdleStateEvent;
+
 /**
  * netty client request handler
  */
@@ -46,6 +54,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
      */
     private final NettyRemotingClient nettyRemotingClient;
 
+    private static byte[] heartBeatData = "heart_beat".getBytes();
+
     /**
      * callback thread executor
      */
@@ -61,7 +71,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
      */
     private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS);
 
-    public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor){
+    public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) {
         this.nettyRemotingClient = nettyRemotingClient;
         this.callbackExecutor = callbackExecutor;
         this.processors = new ConcurrentHashMap();
@@ -89,7 +99,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
      */
     @Override
     public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
-        processReceived(ctx.channel(), (Command)msg);
+        processReceived(ctx.channel(), (Command) msg);
     }
 
     /**
@@ -111,7 +121,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
      */
     public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
         ExecutorService executorRef = executor;
-        if(executorRef == null){
+        if (executorRef == null) {
             executorRef = defaultExecutor;
         }
         this.processors.putIfAbsent(commandType, new Pair<>(processor, executorRef));
@@ -124,20 +134,20 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
      */
     private void processReceived(final Channel channel, final Command command) {
         ResponseFuture future = ResponseFuture.getFuture(command.getOpaque());
-        if(future != null){
+        if (future != null) {
             future.setResponseCommand(command);
             future.release();
-            if(future.getInvokeCallback() != null){
+            if (future.getInvokeCallback() != null) {
                 this.callbackExecutor.submit(new Runnable() {
                     @Override
                     public void run() {
                         future.executeInvokeCallback();
                     }
                 });
-            } else{
+            } else {
                 future.putResponse(command);
             }
-        } else{
+        } else {
             processByCommandType(channel, command);
         }
     }
@@ -164,6 +174,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
 
     /**
      * caught exception
+     *
      * @param ctx   channel handler context
      * @param cause cause
      * @throws Exception
@@ -175,4 +186,18 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter {
         ctx.channel().close();
     }
 
+    @Override
+    public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+        if (evt instanceof IdleStateEvent) {
+            Command heartBeat = new Command();
+            heartBeat.setType(CommandType.HEART_BEAT);
+            heartBeat.setBody(heartBeatData);
+            ctx.writeAndFlush(heartBeat)
+                .addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
+
+        } else {
+            super.userEventTriggered(ctx, evt);
+        }
+    }
+
 }

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java

@@ -17,20 +17,28 @@
 
 package org.apache.dolphinscheduler.remote.handler;
 
-import io.netty.channel.*;
 import org.apache.dolphinscheduler.remote.NettyRemotingServer;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 import org.apache.dolphinscheduler.remote.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelConfig;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.timeout.IdleStateEvent;
+
+
 /**
  * netty server request handler
  */
@@ -49,13 +57,14 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
      */
     private final ConcurrentHashMap<CommandType, Pair<NettyRequestProcessor, ExecutorService>> processors = new ConcurrentHashMap();
 
-    public NettyServerHandler(NettyRemotingServer nettyRemotingServer){
+    public NettyServerHandler(NettyRemotingServer nettyRemotingServer) {
         this.nettyRemotingServer = nettyRemotingServer;
     }
 
     /**
      * When the current channel is not active,
      * the current channel has reached the end of its life cycle
+     *
      * @param ctx channel handler context
      * @throws Exception
      */
@@ -73,7 +82,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
      */
     @Override
     public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
-        processReceived(ctx.channel(), (Command)msg);
+        processReceived(ctx.channel(), (Command) msg);
     }
 
     /**
@@ -95,7 +104,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
      */
     public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) {
         ExecutorService executorRef = executor;
-        if(executorRef == null){
+        if (executorRef == null) {
             executorRef = nettyRemotingServer.getDefaultExecutor();
         }
         this.processors.putIfAbsent(commandType, new Pair<>(processor, executorRef));
@@ -103,6 +112,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
 
     /**
      * process received logic
+     *
      * @param channel channel
      * @param msg     message
      */
@@ -140,7 +150,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
      */
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-        logger.error("exceptionCaught : {}",cause.getMessage(), cause);
+        logger.error("exceptionCaught : {}", cause.getMessage(), cause);
         ctx.channel().close();
     }
 
@@ -170,4 +180,13 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter {
             config.setAutoRead(true);
         }
     }
+
+    @Override
+    public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+        if (evt instanceof IdleStateEvent) {
+            ctx.channel().close();
+        } else {
+            super.userEventTriggered(ctx, evt);
+        }
+    }
 }

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java

@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.remote.utils;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
-
 /**
  *  constant
  */
@@ -29,6 +28,10 @@ public class Constants {
 
     public static final String SLASH = "/";
 
+    public static final int NETTY_SERVER_HEART_BEAT_TIME = 1000 * 60 * 3 + 1000;
+
+    public static final int NETTY_CLIENT_HEART_BEAT_TIME = 1000 * 60;
+
     /**
      *  charset
      */

dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java

@@ -0,0 +1,7 @@
+package org.apache.dolphinscheduler.remote.utils;
+
+/**
+ * Created by Administrator on 2020/10/14.
+ */
+public class NettyUtils {
+}