From 55cc77226f3a915bc59870803c077271afcd0de2 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Wed, 23 Aug 2023 09:43:17 +0800 Subject: [PATCH] [RPC] Refactor the remote module. (#14776) * Add dolphinscheduler-extract-base * Add dolphinscheduler-extract-master * Add dolphinscheduler-extract-worker * Add dolphinscheduler-extract-alert --- .github/actions/labeler/labeler.yml | 2 +- .../dolphinscheduler-alert-server/pom.xml | 10 +- ...tProcessor.java => AlertOperatorImpl.java} | 41 +- .../alert/rpc/AlertRpcServer.java | 35 +- .../alert/service/AlertBootstrapService.java | 2 +- .../runner/AlertBootstrapServiceTest.java | 2 +- dolphinscheduler-api/pom.xml | 11 + .../api/aspect/CacheEvictAspect.java | 30 +- .../api/controller/ExecutorController.java | 2 +- .../pause/pause/PauseExecuteFunction.java | 27 +- .../pause/PauseExecuteFunctionBuilder.java | 5 +- .../instance/stop/StopExecuteFunction.java | 30 +- .../stop/StopExecuteFunctionBuilder.java | 5 +- .../api/k8s/K8sClientService.java | 13 +- .../dolphinscheduler/api/k8s/K8sManager.java | 15 +- .../api/rpc/ApiRpcClient.java | 41 -- .../api/service/ExecutorService.java | 2 +- .../api/service/MetricsCleanUpService.java | 2 +- .../api/service/impl/ClusterServiceImpl.java | 3 +- .../api/service/impl/ExecutorServiceImpl.java | 71 ++- .../api/service/impl/LoggerServiceImpl.java | 93 +++- .../impl/MetricsCleanUpServiceImpl.java | 34 +- .../impl/ProcessDefinitionServiceImpl.java | 4 +- .../service/impl/TaskInstanceServiceImpl.java | 54 +- .../api/k8s/K8sManagerTest.java | 3 +- .../api/service/ClusterServiceTest.java | 3 +- .../service/ExecuteFunctionServiceTest.java | 11 - .../api/service/LoggerServiceTest.java | 11 +- .../service/ProcessDefinitionServiceTest.java | 10 +- dolphinscheduler-common/pom.xml | 10 + dolphinscheduler-dist/release-docs/LICENSE | 11 - dolphinscheduler-dist/release-docs/NOTICE | 28 -- .../licenses/LICENSE-byte-buddy.txt | 202 -------- .../licenses/LICENSE-javassist.txt | 469 ------------------ .../licenses/LICENSE-protostuff-api.txt | 202 -------- .../licenses/LICENSE-protostuff-core.txt | 202 -------- .../licenses/LICENSE-protostuff-runtime.txt | 202 -------- .../LICENSE-protostuff.collectionschema.txt | 202 -------- .../licenses/LICENSE-reflections.txt | 13 - .../dolphinscheduler-extract-alert/pom.xml | 39 ++ .../extract/alert/IAlertOperator.java | 31 ++ .../alert/request}/AlertSendRequest.java | 11 +- .../alert/request}/AlertSendResponse.java | 13 +- .../dolphinscheduler-extract-base/pom.xml | 69 +++ .../extract/base/IRpcRequest.java | 7 +- .../extract/base/IRpcResponse.java | 13 +- .../extract/base/NettyClientHandler.java | 97 ++++ .../extract/base}/NettyRemotingClient.java | 148 ++---- .../base}/NettyRemotingClientFactory.java | 9 +- .../extract/base}/NettyRemotingServer.java | 88 +--- .../base/NettyRemotingServerFactory.java | 21 +- .../extract/base/RpcMethod.java | 33 ++ .../extract}/base/RpcService.java | 11 +- .../extract/base/StandardRpcRequest.java | 48 +- .../extract/base/StandardRpcResponse.java | 38 +- .../base/client/BaseRemoteMethodInvoker.java | 42 +- .../base/client/ClientInvocationHandler.java | 58 +++ .../base/client/ClientMethodInvoker.java | 26 + .../base/client/IRpcClientProxyFactory.java | 16 +- .../JdkDynamicRpcClientProxyFactory.java | 51 ++ ...gletonJdkDynamicRpcClientProxyFactory.java | 29 +- .../base/client/SyncClientMethodInvoker.java | 56 +++ .../base}/config/NettyClientConfig.java | 6 +- .../base}/config/NettyServerConfig.java | 21 +- .../exception/MethodInvocationException.java | 30 ++ .../base/exception}/RemoteException.java | 2 +- .../base/exception}/RemotingException.java | 2 +- .../exception}/RemotingTimeoutException.java | 2 +- .../RemotingTooMuchRequestException.java | 2 +- .../extract/base}/future/InvokeCallback.java | 2 +- .../base}/future/ReleaseSemaphore.java | 2 +- .../extract/base}/future/ResponseFuture.java | 72 +-- .../base/protocal/HeartBeatTransporter.java | 25 +- .../extract/base/protocal/Transporter.java | 55 ++ .../base/protocal/TransporterDecoder.java | 94 ++++ .../base/protocal/TransporterEncoder.java | 47 +- .../base/protocal/TransporterHeader.java | 63 +++ .../base/serialize}/JsonSerializer.java | 19 +- .../base/server/JdkDynamicServerHandler.java | 169 +++++++ .../base/server/ServerMethodInvoker.java | 9 +- .../base/server/ServerMethodInvokerImpl.java | 46 ++ .../SpringServerMethodInvokerDiscovery.java | 66 +++ .../utils/CallerThreadExecutePolicy.java | 2 +- .../extract/base}/utils/ChannelUtils.java | 2 +- .../extract/base}/utils/Constants.java | 2 +- .../extract/base}/utils/Host.java | 2 +- .../base}/utils/NamedThreadFactory.java | 2 +- .../extract/base}/utils/NettyUtils.java | 2 +- .../dolphinscheduler-extract-master/pom.xml | 45 ++ .../master/ILogicTaskInstanceOperator.java | 50 ++ .../extract/master/IMasterCacheService.java | 30 ++ .../extract/master/IMasterLogService.java | 38 ++ .../master/IStreamingTaskOperator.java | 31 ++ .../ITaskInstanceExecutionEventListener.java | 43 ++ .../master/IWorkflowInstanceService.java | 21 +- .../master}/dto/TaskInstanceExecuteDto.java | 2 +- .../master}/dto/WorkflowExecuteDto.java | 2 +- .../transportor}/CacheExpireRequest.java | 13 +- .../ITaskInstanceExecutionEvent.java | 29 +- .../transportor/LogicTaskDispatchRequest.java | 17 +- .../LogicTaskDispatchResponse.java | 42 ++ ...gicTaskInstanceLogFileDownloadRequest.java | 32 ++ ...icTaskInstanceLogFileDownloadResponse.java | 30 ++ .../LogicTaskInstanceLogPageQueryRequest.java | 18 +- ...LogicTaskInstanceLogPageQueryResponse.java | 31 ++ .../transportor/LogicTaskKillRequest.java | 16 +- .../transportor/LogicTaskKillResponse.java | 24 +- .../transportor/LogicTaskPauseRequest.java | 18 +- .../transportor/LogicTaskPauseResponse.java | 25 +- .../StreamingTaskTriggerRequest.java | 23 +- .../StreamingTaskTriggerResponse.java | 39 ++ .../TaskInstanceExecutionFinishEvent.java | 59 +++ .../TaskInstanceExecutionInfoEvent.java | 33 +- .../TaskInstanceExecutionRunningEvent.java | 34 +- .../TaskInstanceForceStartRequest.java | 19 +- .../TaskInstanceForceStartResponse.java | 27 +- .../TaskInstanceWakeupRequest.java | 19 +- .../TaskInstanceWakeupResponse.java | 23 +- .../WorkflowInstanceStateChangeEvent.java | 26 +- .../dolphinscheduler-extract-worker/pom.xml | 45 ++ .../IStreamingTaskInstanceOperator.java | 31 ++ ...TaskInstanceExecutionEventAckListener.java | 39 ++ .../extract/worker/ITaskInstanceOperator.java | 46 ++ .../extract/worker/IWorkerLogService.java | 43 ++ .../worker/transportor/GetAppIdRequest.java | 17 +- .../worker/transportor/GetAppIdResponse.java | 33 ++ .../TaskInstanceDispatchRequest.java | 34 ++ .../TaskInstanceDispatchResponse.java | 42 ++ .../TaskInstanceExecutionFinishEventAck.java | 26 +- .../TaskInstanceExecutionInfoEventAck.java | 24 +- .../TaskInstanceExecutionRunningEventAck.java | 24 +- .../transportor/TaskInstanceKillRequest.java | 15 +- .../transportor/TaskInstanceKillResponse.java | 73 +++ .../TaskInstanceLogFileDownloadRequest.java | 19 +- .../TaskInstanceLogFileDownloadResponse.java | 30 ++ .../TaskInstanceLogPageQueryRequest.java | 18 +- .../TaskInstanceLogPageQueryResponse.java | 31 ++ .../transportor/TaskInstancePauseRequest.java | 30 ++ .../TaskInstancePauseResponse.java | 41 ++ .../TaskInstanceTriggerSavepointRequest.java | 31 ++ .../TaskInstanceTriggerSavepointResponse.java | 27 +- .../UpdateWorkflowHostRequest.java | 33 ++ .../UpdateWorkflowHostResponse.java | 40 ++ dolphinscheduler-extract/pom.xml | 39 ++ dolphinscheduler-master/pom.xml | 11 + .../server/master/MasterServer.java | 12 +- .../server/master/config/MasterConfig.java | 8 - .../controller/WorkflowExecuteController.java | 2 +- .../dispatch/context/ExecutionContext.java | 18 +- .../executor/AbstractExecutorManager.java | 46 -- .../dispatch/executor/ExecutorManager.java | 58 --- .../executor/NettyExecutorManager.java | 187 ------- .../dispatch/host/CommonHostManager.java | 2 +- .../master/dispatch/host/HostManager.java | 2 +- .../dispatch/host/LowerWeightHostManager.java | 2 +- .../dispatch/host/assign/HostWeight.java | 4 +- .../dispatch/host/assign/HostWorker.java | 2 +- .../master/event/TaskDelayEventHandler.java | 16 +- .../master/event/TaskResultEventHandler.java | 22 +- .../master/event/TaskRunningEventHandler.java | 17 +- .../master/event/TaskStateEventHandler.java | 3 +- .../event/TaskUpdatePidEventHandler.java | 14 +- .../metrics/ProcessInstanceMetrics.java | 4 +- .../MasterTaskDispatchProcessor.java | 135 ----- .../processor/TaskExecuteResultProcessor.java | 73 --- .../TaskExecuteRunningProcessor.java | 68 --- .../processor/TaskExecuteStartProcessor.java | 78 --- .../processor/TaskKillResponseProcessor.java | 58 --- .../processor/TaskUpdateRuntimeProcessor.java | 68 --- ...WorkflowExecutingDataRequestProcessor.java | 67 --- .../WorkflowMetricsCleanUpProcessor.java | 48 -- .../queue/StateEventResponseService.java | 31 +- .../master/processor/queue/TaskEvent.java | 27 +- .../processor/queue/TaskExecuteRunnable.java | 7 +- .../registry/MasterWaitingStrategy.java | 4 +- .../master/registry/ServerNodeManager.java | 2 +- .../rpc/ITaskInstanceOperationFunction.java | 7 +- ...TaskInstanceDispatchOperationFunction.java | 96 ++++ ...icITaskInstanceKillOperationFunction.java} | 47 +- ...cITaskInstancePauseOperationFunction.java} | 42 +- ...cTaskInstanceOperationFunctionManager.java | 60 +++ .../rpc/LogicTaskInstanceOperatorImpl.java | 72 +++ .../MasterCacheServiceImpl.java} | 37 +- .../master/rpc/MasterLogServiceImpl.java | 69 +++ .../server/master/rpc/MasterRPCServer.java | 68 --- .../server/master/rpc/MasterRpcClient.java | 63 --- .../server/master/rpc/MasterRpcServer.java | 50 ++ .../master/rpc/StreamingTaskOperatorImpl.java | 58 +++ ...kInstanceForceStartOperationFunction.java} | 42 +- .../master/rpc/TaskInstanceListenerImpl.java | 64 +++ .../TaskInstanceWakeupOperationFunction.java} | 46 +- .../rpc/WorkflowInstanceServiceImpl.java | 52 ++ .../master/runner/BaseTaskDispatcher.java | 38 +- .../runner/StateWheelExecuteThread.java | 14 +- .../runner/StreamTaskExecuteRunnable.java | 43 +- .../master/runner/WorkflowEventLooper.java | 7 +- .../runner/WorkflowExecuteRunnable.java | 73 ++- .../WorkflowExecuteRunnableFactory.java | 5 - .../runner/WorkflowExecuteThreadPool.java | 25 +- .../dispatcher/MasterTaskDispatcher.java | 34 +- .../dispatcher/WorkerTaskDispatcher.java | 32 +- .../AsyncMasterDelayTaskExecuteRunnable.java | 6 +- ...MasterDelayTaskExecuteRunnableFactory.java | 6 +- .../AsyncMasterTaskDelayQueueLooper.java | 12 +- .../execute/DefaultTaskExecuteRunnable.java | 16 +- .../DefaultTaskExecuteRunnableFactory.java | 6 +- .../MasterDelayTaskExecuteRunnable.java | 6 +- .../execute/MasterTaskExecuteRunnable.java | 17 +- .../SyncMasterDelayTaskExecuteRunnable.java | 6 +- ...MasterDelayTaskExecuteRunnableFactory.java | 6 +- ...kInstanceExecutionEventListenFunction.java | 24 + ...ExecutionEventListenerFunctionManager.java | 57 +++ ...anceExecutionInfoEventListenFunction.java} | 43 +- ...nceExecutionResultEventListenFunction.java | 52 ++ ...ceExecutionRunningEventListenFunction.java | 43 ++ ...TaskInstanceStateEventListenFunction.java} | 49 +- ...askInstanceExecuteRunningEventSender.java} | 39 +- ...ogicTaskInstanceExecutionEventSender.java} | 16 +- ...kInstanceExecutionEventSenderManager.java} | 21 +- ...skInstanceExecutionFinishEventSender.java} | 38 +- .../MasterTaskUpdateRuntimeMessageSender.java | 64 --- ...seTaskExecuteRunnableDispatchOperator.java | 35 ++ ... BaseTaskExecuteRunnableKillOperator.java} | 33 +- .../BaseTaskExecuteRunnablePauseOperator.java | 38 ++ ...seTaskExecuteRunnableTimeoutOperator.java} | 29 +- ...icTaskExecuteRunnableDispatchOperator.java | 31 ++ .../LogicTaskExecuteRunnableKillOperator.java | 56 +++ ...LogicTaskExecuteRunnablePauseOperator.java | 52 ++ ...gicTaskExecuteRunnableTimeoutOperator.java | 59 +++ ... TaskExecuteRunnableDispatchOperator.java} | 4 +- .../TaskExecuteRunnableKillOperator.java | 54 ++ ....java => TaskExecuteRunnableOperator.java} | 4 +- .../TaskExecuteRunnableOperatorManager.java | 81 +++ ... => TaskExecuteRunnablePauseOperator.java} | 31 +- .../TaskExecuteRunnableTimeoutOperator.java | 57 +++ .../runner/task/dynamic/DynamicLogicTask.java | 24 +- .../DynamicLogicTaskPluginFactory.java | 6 +- .../subworkflow/SubWorkflowLogicTask.java | 26 +- .../SubWorkflowLogicTaskPluginFactory.java | 6 +- .../master/service/ExecutingService.java | 4 +- .../master/service/MasterFailoverService.java | 7 +- .../master/service/WorkerFailoverService.java | 98 ++-- .../dispatch/ExecutionContextTestUtils.java | 63 --- .../executor/NettyExecutorManagerTest.java | 100 ---- .../host/RoundRobinHostManagerTest.java | 69 --- .../master/processor/CacheProcessorTest.java | 69 --- .../processor/TaskAckProcessorTest.java | 89 ---- .../TaskKillResponseProcessorTest.java | 70 --- .../queue/TaskResponseServiceTest.java | 120 ----- .../runner/WorkflowExecuteRunnableTest.java | 3 - .../dispatcher/MasterTaskDispatcherTest.java | 6 +- .../dispatcher/WorkerTaskDispatcherTest.java | 6 +- .../PriorityTaskExecuteRunnableTest.java | 4 +- .../task/dynamic/DynamicLogicTaskTest.java | 6 - .../master/service/FailoverServiceTest.java | 5 - dolphinscheduler-microbench/pom.xml | 4 - .../microbench/common/RpcTest.java | 78 --- dolphinscheduler-remote/pom.xml | 92 ---- .../remote/codec/NettyDecoder.java | 146 ------ .../remote/command/BaseMessage.java | 57 --- .../remote/command/Message.java | 124 ----- .../remote/command/MessageContext.java | 56 --- .../remote/command/MessageHeader.java | 55 -- .../remote/command/MessageType.java | 116 ----- .../remote/command/RequestMessageBuilder.java | 43 -- .../command/ResponseMessageBuilder.java | 35 -- .../command/log/GetLogBytesResponse.java | 45 -- .../command/log/RemoveTaskLogRequest.java | 44 -- .../command/log/RemoveTaskLogResponse.java | 44 -- .../command/log/RollViewLogResponse.java | 44 -- .../task/TaskExecuteResultMessage.java | 100 ---- .../task/TaskExecuteResultMessageAck.java | 55 -- .../task/TaskExecuteRunningMessage.java | 92 ---- .../task/TaskExecuteRunningMessageAck.java | 56 --- .../task/TaskUpdateRuntimeMessage.java | 76 --- .../task/WorkflowHostChangeResponse.java | 50 -- .../WorkflowExecutingDataRequest.java | 41 -- .../WorkflowExecutingDataResponse.java | 42 -- .../factory/NettyRemotingServerFactory.java | 33 -- .../remote/handler/NettyClientHandler.java | 197 -------- .../remote/handler/NettyServerHandler.java | 188 ------- .../remote/processor/GetAppIdProcessor.java | 63 --- .../processor/GetLogBytesProcessor.java | 51 -- .../remote/processor/NettyRemoteChannel.java | 82 --- .../processor/NettyRequestProcessor.java | 37 -- .../processor/RemoveTaskLogProcessor.java | 62 --- .../remote/processor/RollViewLogRequest.java | 58 --- .../processor/StateEventCallbackService.java | 149 ------ .../processor/ViewWholeLogProcessor.java | 51 -- .../apache/dolphinscheduler/rpc/base/Rpc.java | 54 -- .../rpc/client/ConsumerConfig.java | 87 ---- .../rpc/client/ConsumerConfigCache.java | 40 -- .../rpc/client/ConsumerInterceptor.java | 120 ----- .../rpc/client/RpcClient.java | 42 -- .../rpc/client/RpcRequestCache.java | 46 -- .../rpc/client/RpcRequestTable.java | 52 -- .../rpc/codec/NettyDecoder.java | 83 ---- .../rpc/codec/NettyEncoder.java | 53 -- .../rpc/common/ConsumerConfigConstants.java | 34 -- .../rpc/common/RequestEventType.java | 41 -- .../rpc/common/RpcRequest.java | 84 ---- .../rpc/common/RpcResponse.java | 66 --- .../rpc/common/ThreadPoolManager.java | 45 -- .../rpc/config/ServiceBean.java | 70 --- .../rpc/future/RpcFuture.java | 87 ---- .../rpc/protocol/EventType.java | 42 -- .../rpc/protocol/MessageHeader.java | 77 --- .../rpc/remote/NettyClient.java | 255 ---------- .../rpc/remote/NettyClientHandler.java | 117 ----- .../rpc/remote/NettyServer.java | 205 -------- .../rpc/remote/NettyServerHandler.java | 111 ----- .../rpc/serializer/ProtoStuffSerializer.java | 62 --- .../rpc/serializer/ProtoStuffUtils.java | 68 --- .../rpc/serializer/RpcSerializer.java | 50 -- .../remote/NettyRemotingClientTest.java | 222 --------- .../command/alert/AlertSendRequestTest.java | 44 -- .../command/alert/AlertSendResponseTest.java | 49 -- .../command/cache/CacheExpireRequestTest.java | 35 -- .../command/future/ResponseFutureTest.java | 63 --- .../command/log/GetLogBytesRequestTest.java | 34 -- .../command/log/GetLogBytesResponseTest.java | 35 -- .../command/log/RemoveTaskLogRequestTest.java | 34 -- .../log/RemoveTaskLogResponseTest.java | 34 -- .../command/log/RollViewLogRequestTest.java | 36 -- .../remote/utils/HostTest.java | 41 -- .../dolphinscheduler/rpc/IUserService.java | 34 -- .../apache/dolphinscheduler/rpc/RpcTest.java | 52 -- .../dolphinscheduler/rpc/UserCallback.java | 37 -- .../dolphinscheduler/rpc/UserService.java | 52 -- dolphinscheduler-service/pom.xml | 16 +- .../service/alert/AlertClientService.java | 114 ----- .../cache/impl/CacheNotifyServiceImpl.java | 134 ----- .../service/log/LogClient.java | 243 --------- .../service/process/ProcessServiceImpl.java | 38 +- .../service/utils/ProcessUtils.java | 10 +- .../service/alert/AlertClientServiceTest.java | 168 ------- .../service/cache/CacheNotifyServiceTest.java | 93 ---- .../log/LoggerRequestProcessorTest.java | 115 ----- .../task/api/AbstractCommandExecutor.java | 15 +- .../task/api/k8s/impl/K8sTaskExecutor.java | 11 +- .../plugin/task/api/utils/LogUtils.java | 18 +- .../plugin/task/api/utils/TaskUtils.java | 50 ++ dolphinscheduler-worker/pom.xml | 20 +- .../server/worker/WorkerServer.java | 19 +- .../server/worker/config/WorkerConfig.java | 5 - .../worker/message/MessageRetryRunner.java | 51 +- ... => TaskInstanceExecutionEventSender.java} | 13 +- ...skInstanceExecutionFinishEventSender.java} | 40 +- ...stanceExecutionInfoUpdateEventSender.java} | 40 +- ...kInstanceExecutionRunningEventSender.java} | 40 +- .../WorkerTaskDispatchProcessor.java | 142 ------ .../WorkerTaskExecuteResultAckProcessor.java | 76 --- .../WorkerTaskExecuteRunningAckProcessor.java | 71 --- .../processor/WorkerTaskPauseProcessor.java | 46 -- .../WorkerTaskSavePointProcessor.java | 134 ----- .../WorkerTaskUpdatePidAckProcessor.java | 73 --- .../WorkflowHostChangeProcessor.java | 92 ---- .../worker/registry/WorkerRegistryClient.java | 2 +- .../registry/WorkerWaitingStrategy.java | 13 +- .../StreamingTaskInstanceOperatorImpl.java | 81 +++ ...InstanceExecutionEventAckListenerImpl.java | 55 ++ .../worker/rpc/TaskInstanceOperatorImpl.java | 62 +++ .../worker/rpc/WorkerLogServiceImpl.java | 87 ++++ .../worker/rpc/WorkerMessageSender.java | 44 +- .../server/worker/rpc/WorkerRpcClient.java | 70 --- .../server/worker/rpc/WorkerRpcServer.java | 41 +- ...DefaultWorkerDelayTaskExecuteRunnable.java | 3 - ...WorkerDelayTaskExecuteRunnableFactory.java | 4 - .../GlobalTaskInstanceDispatchQueue.java | 70 +++ ...GlobalTaskInstanceDispatchQueueLooper.java | 113 +++++ .../worker/runner/TaskCallbackImpl.java | 8 +- .../WorkerDelayTaskExecuteRunnable.java | 3 - ...WorkerDelayTaskExecuteRunnableFactory.java | 4 - .../worker/runner/WorkerManagerThread.java | 16 +- .../runner/WorkerTaskExecuteRunnable.java | 43 +- ...rkerTaskExecuteRunnableFactoryBuilder.java | 3 - ...stanceExecutionEventAckListenFunction.java | 24 + ...xecutionEventAckListenFunctionManager.java | 50 ++ ...ExecutionFinishEventAckListenFunction.java | 56 +++ ...ceExecutionInfoEventAckListenFunction.java | 57 +++ ...xecutionRunningEventAckListenFunction.java | 55 ++ .../ITaskInstanceOperationFunction.java | 24 + ...TaskInstanceDispatchOperationFunction.java | 73 +++ .../TaskInstanceKillOperationFunction.java} | 101 +--- .../TaskInstanceOperationFunctionManager.java | 54 ++ .../TaskInstancePauseOperationFunction.java | 45 ++ .../UpdateWorkflowHostOperationFunction.java | 67 +++ .../WorkerTaskDispatchProcessorTest.java | 99 ---- ...ultWorkerDelayTaskExecuteRunnableTest.java | 5 - pom.xml | 28 +- tools/dependencies/known-dependencies.txt | 7 - 391 files changed, 5916 insertions(+), 12312 deletions(-) rename dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/{AlertRequestProcessor.java => AlertOperatorImpl.java} (50%) delete mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/rpc/ApiRpcClient.java delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-byte-buddy.txt delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-alert/pom.xml create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/IAlertOperator.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert => dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request}/AlertSendRequest.java (73%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert => dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request}/AlertSendResponse.java (79%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/MasterRpcProcessor.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcRequest.java (85%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/BaseLogProcessor.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcResponse.java (81%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/NettyRemotingClient.java (67%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/NettyRemotingClientFactory.java (71%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/NettyRemotingServer.java (74%) rename dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java (68%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract}/base/RpcService.java (85%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcRequest.java (55%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchResponse.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcResponse.java (50%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java (50%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java (68%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskPauseRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java (54%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/config/NettyClientConfig.java (91%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/config/NettyServerConfig.java (75%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/MethodInvocationException.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception}/RemoteException.java (95%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception}/RemotingException.java (96%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception}/RemotingTimeoutException.java (95%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception}/RemotingTooMuchRequestException.java (94%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/future/InvokeCallback.java (94%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/future/ReleaseSemaphore.java (95%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/future/ResponseFuture.java (73%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/HeartBeatTransporter.java (62%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/Transporter.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterDecoder.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterEncoder.java (51%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterHeader.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize}/JsonSerializer.java (91%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java rename dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java (81%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/utils/CallerThreadExecutePolicy.java (95%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/utils/ChannelUtils.java (97%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/utils/Constants.java (96%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/utils/Host.java (96%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/utils/NamedThreadFactory.java (96%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base}/utils/NettyUtils.java (96%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterCacheService.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterLogService.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IStreamingTaskOperator.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskInstanceExecutionEventListener.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowMetricsCleanUpRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java (63%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master}/dto/TaskInstanceExecuteDto.java (97%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master}/dto/WorkflowExecuteDto.java (98%) rename {dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/cache => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor}/CacheExpireRequest.java (74%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskInstanceExecutionEvent.java (64%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchRequest.java (71%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchResponse.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadRequest.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadResponse.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponse.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryRequest.java (73%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryResponse.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillRequest.java (75%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponse.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillResponse.java (69%) rename dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseRequest.java (75%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseResponse.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteStartMessage.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java (63%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionFinishEvent.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionInfoEvent.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillResponse.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionRunningEvent.java (68%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskWakeupRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskForceStartRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java (66%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupResponse.java (68%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowStateEventChangeRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/WorkflowInstanceStateChangeEvent.java (68%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IStreamingTaskInstanceOperator.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IWorkerLogService.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdRequest.java (73%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdResponse.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchRequest.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeAckMessage.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionInfoEventAck.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointResponse.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java (67%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java (74%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java rename dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadRequest.java (71%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadResponse.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryRequest.java (71%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryResponse.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseRequest.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseResponse.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequest.java => dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java (67%) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostRequest.java create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostResponse.java create mode 100644 dolphinscheduler-extract/pom.xml delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskDispatchProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResultProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskUpdateRuntimeProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowMetricsCleanUpProcessor.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/WorkerRpcProcessor.java => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/ITaskInstanceOperationFunction.java (85%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{processor/MasterTaskKillProcessor.java => rpc/LogicITaskInstanceKillOperationFunction.java} (62%) rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{processor/MasterTaskPauseProcessor.java => rpc/LogicITaskInstancePauseOperationFunction.java} (55%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{processor/CacheProcessor.java => rpc/MasterCacheServiceImpl.java} (59%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/StreamingTaskOperatorImpl.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{processor/TaskForceStartProcessor.java => rpc/TaskInstanceForceStartOperationFunction.java} (54%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceListenerImpl.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{processor/TaskWakeupProcessor.java => rpc/TaskInstanceWakeupOperationFunction.java} (53%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/ITaskInstanceExecutionEventListenFunction.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionEventListenerFunctionManager.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/{operator/TaskOperatorManager.java => listener/TaskInstanceExecutionInfoEventListenFunction.java} (53%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionResultEventListenFunction.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionRunningEventListenFunction.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{processor/StateEventProcessor.java => runner/listener/TaskInstanceStateEventListenFunction.java} (64%) rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/{MasterTaskExecuteRunningMessageSender.java => LogicTaskInstanceExecuteRunningEventSender.java} (56%) rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/{MasterMessageSender.java => LogicTaskInstanceExecutionEventSender.java} (71%) rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/{MasterMessageSenderManager.java => LogicTaskInstanceExecutionEventSenderManager.java} (58%) rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/{MasterTaskExecuteResultMessageSender.java => LogicTaskInstanceExecutionFinishEventSender.java} (60%) delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskUpdateRuntimeMessageSender.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/{TaskKillOperator.java => BaseTaskExecuteRunnableKillOperator.java} (66%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnablePauseOperator.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/{TaskTimeoutOperator.java => BaseTaskExecuteRunnableTimeoutOperator.java} (74%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableDispatchOperator.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableKillOperator.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnablePauseOperator.java create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableTimeoutOperator.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/{TaskDispatchOperator.java => TaskExecuteRunnableDispatchOperator.java} (89%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableKillOperator.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/{TaskOperator.java => TaskExecuteRunnableOperator.java} (89%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/{TaskPauseOperator.java => TaskExecuteRunnablePauseOperator.java} (52%) create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableTimeoutOperator.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java delete mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java delete mode 100644 dolphinscheduler-remote/pom.xml delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseMessage.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Message.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageContext.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageHeader.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageType.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/RequestMessageBuilder.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ResponseMessageBuilder.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponse.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponse.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponse.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessage.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessageAck.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessage.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessageAck.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeMessage.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeResponse.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataRequest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataResponse.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingServerFactory.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetAppIdProcessor.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetLogBytesProcessor.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RemoveTaskLogProcessor.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RollViewLogRequest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/ViewWholeLogProcessor.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequestTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java delete mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java delete mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheNotifyServiceImpl.java delete mode 100644 dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java delete mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java delete mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java delete mode 100644 dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java rename dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/{MessageSender.java => TaskInstanceExecutionEventSender.java} (70%) rename dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/{TaskExecuteResultMessageSender.java => TaskInstanceExecutionFinishEventSender.java} (60%) rename dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/{WorkerTaskUpdateRuntimeMessageSender.java => TaskInstanceExecutionInfoUpdateEventSender.java} (50%) rename dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/{TaskExecuteRunningMessageSender.java => TaskInstanceExecutionRunningEventSender.java} (55%) delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskDispatchProcessor.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteResultAckProcessor.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteRunningAckProcessor.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskPauseProcessor.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskSavePointProcessor.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskUpdatePidAckProcessor.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkflowHostChangeProcessor.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/StreamingTaskInstanceOperatorImpl.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceExecutionEventAckListenerImpl.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceOperatorImpl.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerLogServiceImpl.java delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueue.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueueLooper.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/ITaskInstanceExecutionEventAckListenFunction.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionEventAckListenFunctionManager.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionFinishEventAckListenFunction.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionInfoEventAckListenFunction.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionRunningEventAckListenFunction.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceDispatchOperationFunction.java rename dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/{processor/WorkerTaskKillProcessor.java => runner/operator/TaskInstanceKillOperationFunction.java} (59%) create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionManager.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstancePauseOperationFunction.java create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/UpdateWorkflowHostOperationFunction.java delete mode 100644 dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskDispatchProcessorTest.java diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml index fbfcb098fe..b4776198e2 100644 --- a/.github/actions/labeler/labeler.yml +++ b/.github/actions/labeler/labeler.yml @@ -25,7 +25,7 @@ backend: - 'dolphinscheduler-dist/**/*' - 'dolphinscheduler-master/**/*' - 'dolphinscheduler-registry/**/*' - - 'dolphinscheduler-remote/**/*' + - 'dolphinscheduler-extract/**/*' - 'dolphinscheduler-scheduler-plugin/**/*' - 'dolphinscheduler-service/**/*' - 'dolphinscheduler-spi/**/*' diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml index 4ced9ea953..507d7acb45 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml @@ -28,10 +28,7 @@ ${project.artifactId} - - org.apache.dolphinscheduler - dolphinscheduler-remote - + org.apache.dolphinscheduler dolphinscheduler-meter @@ -51,6 +48,11 @@ dolphinscheduler-registry-all + + org.apache.dolphinscheduler + dolphinscheduler-extract-alert + + com.google.guava guava diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRequestProcessor.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java similarity index 50% rename from dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRequestProcessor.java rename to dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java index f0141a4108..36c0a5556d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRequestProcessor.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java @@ -14,49 +14,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.dolphinscheduler.alert.rpc; import org.apache.dolphinscheduler.alert.service.AlertBootstrapService; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequest; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.JsonSerializer; +import org.apache.dolphinscheduler.extract.alert.IAlertOperator; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; -@Component @Slf4j -public final class AlertRequestProcessor implements NettyRequestProcessor { +@Service +public class AlertOperatorImpl implements IAlertOperator { - private final AlertBootstrapService alertBootstrapService; - - public AlertRequestProcessor(AlertBootstrapService alertBootstrapService) { - this.alertBootstrapService = alertBootstrapService; - } + @Autowired + private AlertBootstrapService alertBootstrapService; @Override - public void process(Channel channel, Message message) { - AlertSendRequest alertSendRequest = JsonSerializer.deserialize(message.getBody(), AlertSendRequest.class); - - log.info("Received command : {}", alertSendRequest); - + public AlertSendResponse sendAlert(AlertSendRequest alertSendRequest) { + log.info("Received AlertSendRequest : {}", alertSendRequest); AlertSendResponse alertSendResponse = alertBootstrapService.syncHandler( alertSendRequest.getGroupId(), alertSendRequest.getTitle(), alertSendRequest.getContent(), alertSendRequest.getWarnType()); - channel.writeAndFlush(alertSendResponse.convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.ALERT_SEND_REQUEST; + log.info("Handle AlertSendRequest finish: {}", alertSendResponse); + return alertSendResponse; } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java index f5bd470359..4a8c621d30 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java @@ -18,43 +18,32 @@ package org.apache.dolphinscheduler.alert.rpc; import org.apache.dolphinscheduler.alert.config.AlertConfig; -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.factory.NettyRemotingServerFactory; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; - -import java.util.List; +import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Slf4j @Service -public class AlertRpcServer implements AutoCloseable { - - @Autowired - private List nettyRequestProcessors; - @Autowired - private AlertConfig alertConfig; +public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { - private NettyRemotingServer nettyRemotingServer; + public AlertRpcServer(AlertConfig alertConfig) { + super(new NettyRemotingServer(new NettyServerConfig(alertConfig.getPort()))); + } public void start() { - log.info("Starting alert rpc server..."); - nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(alertConfig.getPort()); - for (NettyRequestProcessor nettyRequestProcessor : nettyRequestProcessors) { - nettyRemotingServer.registerProcessor(nettyRequestProcessor); - log.info("Success register netty processor: {}", nettyRequestProcessor.getClass().getName()); - } + log.info("Starting AlertRpcServer..."); nettyRemotingServer.start(); - log.info("Started alert rpc server..."); + log.info("Started AlertRpcServer..."); } @Override - public void close() throws Exception { - log.info("Closing alert rpc server..."); + public void close() { + log.info("Closing AlertRpcServer..."); nettyRemotingServer.close(); - log.info("Closed alert rpc server..."); + log.info("Closed AlertRpcServer..."); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java index 3394f83097..57ef75d6d6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java @@ -37,7 +37,7 @@ import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.AlertSendStatus; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java index 745dbc9865..3819200690 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.PluginDefine; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 62c29d5f78..467826bc41 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -37,6 +37,7 @@ + @@ -82,6 +83,16 @@ dolphinscheduler-storage-all + + org.apache.dolphinscheduler + dolphinscheduler-extract-master + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-worker + + org.codehaus.janino janino diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java index 3d7a4c0afe..cf6df2823a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java @@ -18,10 +18,15 @@ package org.apache.dolphinscheduler.api.aspect; import org.apache.dolphinscheduler.common.enums.CacheType; -import org.apache.dolphinscheduler.remote.command.cache.CacheExpireRequest; -import org.apache.dolphinscheduler.service.cache.CacheNotifyService; +import org.apache.dolphinscheduler.common.model.Server; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.IMasterCacheService; +import org.apache.dolphinscheduler.extract.master.transportor.CacheExpireRequest; +import org.apache.dolphinscheduler.registry.api.RegistryClient; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.service.cache.impl.CacheKeyGenerator; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.lang.reflect.Method; @@ -65,7 +70,7 @@ public class CacheEvictAspect { private CacheKeyGenerator cacheKeyGenerator; @Autowired - private CacheNotifyService cacheNotifyService; + private RegistryClient registryClient; @Pointcut("@annotation(org.springframework.cache.annotation.CacheEvict)") public void cacheEvictPointCut() { @@ -96,7 +101,7 @@ public class CacheEvictAspect { } } if (StringUtils.isNotEmpty(cacheKey)) { - cacheNotifyService.notifyMaster(new CacheExpireRequest(cacheType, cacheKey).convert2Command()); + notifyMaster(cacheType, cacheKey); } } @@ -134,4 +139,21 @@ public class CacheEvictAspect { } return obj.toString(); } + + private void notifyMaster(CacheType cacheType, String cacheKey) { + try { + List serverList = registryClient.getServerList(RegistryNodeType.MASTER); + if (CollectionUtils.isEmpty(serverList)) { + return; + } + for (Server server : serverList) { + IMasterCacheService masterCacheService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(server.getHost() + ":" + server.getPort(), IMasterCacheService.class); + masterCacheService.cacheExpire(new CacheExpireRequest(cacheType, cacheKey)); + } + } catch (Exception e) { + log.error("notify master error", e); + } + + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 5151665b34..cdfabedcae 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -40,7 +40,7 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; import org.apache.commons.lang3.StringUtils; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/pause/PauseExecuteFunction.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/pause/PauseExecuteFunction.java index 4d52cc22aa..5df321628a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/pause/PauseExecuteFunction.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/pause/PauseExecuteFunction.java @@ -20,24 +20,20 @@ package org.apache.dolphinscheduler.api.executor.workflow.instance.pause.pause; import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.executor.ExecuteFunction; import org.apache.dolphinscheduler.api.executor.ExecuteRuntimeException; -import org.apache.dolphinscheduler.api.rpc.ApiRpcClient; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; public class PauseExecuteFunction implements ExecuteFunction { private final ProcessInstanceDao processInstanceDao; - private final ApiRpcClient apiRpcClient; - - public PauseExecuteFunction(ProcessInstanceDao processInstanceDao, ApiRpcClient apiRpcClient) { + public PauseExecuteFunction(ProcessInstanceDao processInstanceDao) { this.processInstanceDao = processInstanceDao; - this.apiRpcClient = apiRpcClient; } @Override @@ -59,15 +55,18 @@ public class PauseExecuteFunction implements ExecuteFunction> createWorkflowInstanceExecuteFunction(ExecuteContext executeContext) { - return CompletableFuture.completedFuture(new PauseExecuteFunction(processInstanceDao, apiRpcClient)); + return CompletableFuture.completedFuture(new PauseExecuteFunction(processInstanceDao)); } @Override diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/stop/StopExecuteFunction.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/stop/StopExecuteFunction.java index a95a57e134..8afb18a278 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/stop/StopExecuteFunction.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/stop/StopExecuteFunction.java @@ -20,14 +20,13 @@ package org.apache.dolphinscheduler.api.executor.workflow.instance.stop; import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.executor.ExecuteFunction; import org.apache.dolphinscheduler.api.executor.ExecuteRuntimeException; -import org.apache.dolphinscheduler.api.rpc.ApiRpcClient; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import lombok.extern.slf4j.Slf4j; @@ -35,12 +34,9 @@ import lombok.extern.slf4j.Slf4j; public class StopExecuteFunction implements ExecuteFunction { private final ProcessInstanceDao processInstanceDao; - // todo: Use ApiRpcClient instead of NettyRemotingClient - private final ApiRpcClient apiRpcClient; - public StopExecuteFunction(ProcessInstanceDao processInstanceDao, ApiRpcClient apiRpcClient) { + public StopExecuteFunction(ProcessInstanceDao processInstanceDao) { this.processInstanceDao = processInstanceDao; - this.apiRpcClient = apiRpcClient; } @Override @@ -60,17 +56,17 @@ public class StopExecuteFunction implements ExecuteFunction> createWorkflowInstanceExecuteFunction(ExecuteContext executeContext) { - return CompletableFuture.completedFuture(new StopExecuteFunction(processInstanceDao, apiRpcClient)); + return CompletableFuture.completedFuture(new StopExecuteFunction(processInstanceDao)); } @Override diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java index 76b875ad77..b30a3cb37f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.api.k8s; import org.apache.dolphinscheduler.dao.entity.K8sNamespace; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import java.util.Optional; @@ -43,16 +42,16 @@ public class K8sClientService { private K8sManager k8sManager; public ResourceQuota upsertNamespaceAndResourceToK8s(K8sNamespace k8sNamespace, - String yamlStr) throws RemotingException { + String yamlStr) { if (!checkNamespaceToK8s(k8sNamespace.getNamespace(), k8sNamespace.getClusterCode())) { - throw new RemotingException(String.format( + throw new RuntimeException(String.format( "namespace %s does not exist in k8s cluster, please create namespace in k8s cluster first", k8sNamespace.getNamespace())); } return upsertNamespacedResourceToK8s(k8sNamespace, yamlStr); } - public Optional deleteNamespaceToK8s(String name, Long clusterCode) throws RemotingException { + public Optional deleteNamespaceToK8s(String name, Long clusterCode) { Optional result = getNamespaceFromK8s(name, clusterCode); if (result.isPresent()) { KubernetesClient client = k8sManager.getK8sClient(clusterCode); @@ -67,7 +66,7 @@ public class K8sClientService { } private ResourceQuota upsertNamespacedResourceToK8s(K8sNamespace k8sNamespace, - String yamlStr) throws RemotingException { + String yamlStr) { KubernetesClient client = k8sManager.getK8sClient(k8sNamespace.getClusterCode()); @@ -93,7 +92,7 @@ public class K8sClientService { .createOrReplace(body); } - private Optional getNamespaceFromK8s(String name, Long clusterCode) throws RemotingException { + private Optional getNamespaceFromK8s(String name, Long clusterCode) { NamespaceList listNamespace = k8sManager.getK8sClient(clusterCode).namespaces().list(); @@ -105,7 +104,7 @@ public class K8sClientService { return list; } - private boolean checkNamespaceToK8s(String name, Long clusterCode) throws RemotingException { + private boolean checkNamespaceToK8s(String name, Long clusterCode) { Optional result = getNamespaceFromK8s(name, clusterCode); return result.isPresent(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java index 955b73d467..9c1281f84c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.k8s; import org.apache.dolphinscheduler.dao.entity.Cluster; import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.service.utils.ClusterConfUtils; import java.util.Hashtable; @@ -55,7 +54,7 @@ public class K8sManager { * @param clusterCode * @return */ - public synchronized KubernetesClient getK8sClient(Long clusterCode) throws RemotingException { + public synchronized KubernetesClient getK8sClient(Long clusterCode) { if (null == clusterCode) { return null; } @@ -68,7 +67,7 @@ public class K8sManager { * @return new client if need updated */ public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode, - boolean update) throws RemotingException { + boolean update) { if (null == clusterCode) { return null; } @@ -99,7 +98,7 @@ public class K8sManager { } } - private void createK8sClientInner(Long clusterCode) throws RemotingException { + private void createK8sClientInner(Long clusterCode) { Cluster cluster = clusterMapper.queryByClusterCode(clusterCode); if (cluster == null) { return; @@ -111,20 +110,20 @@ public class K8sManager { try { client = getClient(k8sConfig); clientMap.put(clusterCode, client); - } catch (RemotingException e) { + } catch (Exception e) { log.error("cluster code ={},fail to get k8s ApiClient: {}", clusterCode, e.getMessage()); - throw new RemotingException("fail to get k8s ApiClient:" + e.getMessage()); + throw new RuntimeException("fail to get k8s ApiClient:" + e.getMessage()); } } } - private KubernetesClient getClient(String configYaml) throws RemotingException { + private KubernetesClient getClient(String configYaml) throws RuntimeException { try { Config config = Config.fromKubeconfig(configYaml); return new KubernetesClientBuilder().withConfig(config).build(); } catch (Exception e) { log.error("Fail to get k8s ApiClient", e); - throw new RemotingException("fail to get k8s ApiClient:" + e.getMessage()); + throw new RuntimeException("fail to get k8s ApiClient:" + e.getMessage()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/rpc/ApiRpcClient.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/rpc/ApiRpcClient.java deleted file mode 100644 index f0c1ba6d6d..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/rpc/ApiRpcClient.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.rpc; - -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; - -import org.springframework.stereotype.Component; - -@Component -public class ApiRpcClient { - - private final NettyRemotingClient nettyRemotingClient; - - public ApiRpcClient() { - this.nettyRemotingClient = new NettyRemotingClient(new NettyClientConfig()); - } - - public void send(Host host, Message message) throws RemotingException { - nettyRemotingClient.send(host, message); - } - -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index c67f790c34..408fdee7f3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MetricsCleanUpService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MetricsCleanUpService.java index 9ffa95de1d..a8971d16dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MetricsCleanUpService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MetricsCleanUpService.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.service; public interface MetricsCleanUpService { - void cleanUpWorkflowMetricsByDefinitionCode(String workflowDefinitionCode); + void cleanUpWorkflowMetricsByDefinitionCode(Long workflowDefinitionCode); void cleanUpApiResponseTimeMetricsByUserId(int userId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index 61e06a5841..8c181c00fe 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.dao.entity.K8sNamespace; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.service.utils.ClusterConfUtils; import org.apache.commons.collections4.CollectionUtils; @@ -321,7 +320,7 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic && !config.equals(ClusterConfUtils.getK8sConfig(clusterExist.getConfig()))) { try { k8sManager.getAndUpdateK8sClient(code, true); - } catch (RemotingException e) { + } catch (Exception e) { log.error("Update K8s error.", e); putMsg(result, Status.K8S_CLIENT_OPS_ERROR, name); return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index ba996a46eb..be3a007d26 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -82,16 +82,17 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.IStreamingTaskOperator; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService; +import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteStartMessage; -import org.apache.dolphinscheduler.remote.command.task.TaskForceStartRequest; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowExecutingDataRequest; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowExecutingDataResponse; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; @@ -164,9 +165,6 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ @Autowired private TaskDefinitionLogMapper taskDefinitionLogMapper; - @Autowired - private StateEventCallbackService stateEventCallbackService; - @Autowired private TaskDefinitionMapper taskDefinitionMapper; @@ -655,10 +653,12 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ executionStatus.getDesc(), processInstance.getName()); // directly send the process instance state change event to target master, not guarantee the event send // success - WorkflowStateEventChangeRequest workflowStateEventChangeRequest = new WorkflowStateEventChangeRequest( + WorkflowInstanceStateChangeEvent workflowStateEventChangeRequest = new WorkflowInstanceStateChangeEvent( processInstance.getId(), 0, processInstance.getState(), processInstance.getId(), 0); - Host host = new Host(processInstance.getHost()); - stateEventCallbackService.sendResult(host, workflowStateEventChangeRequest.convert2Command()); + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(processInstance.getHost(), ITaskInstanceExecutionEventListener.class); + iTaskInstanceExecutionEventListener.onWorkflowInstanceInstanceStateChange(workflowStateEventChangeRequest); putMsg(result, Status.SUCCESS); } else { log.error("Process instance state update error, processInstanceName:{}.", processInstance.getName()); @@ -684,9 +684,10 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ taskGroupQueue.setForceStart(Flag.YES.getCode()); processService.updateTaskGroupQueue(taskGroupQueue); log.info("Sending force start command to master: {}.", processInstance.getHost()); - stateEventCallbackService.sendResult( - Host.of(processInstance.getHost()), - new TaskForceStartRequest(processInstance.getId(), taskGroupQueue.getTaskId()).convert2Command()); + ILogicTaskInstanceOperator iLogicTaskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(processInstance.getHost(), ILogicTaskInstanceOperator.class); + iLogicTaskInstanceOperator.forceStartTaskInstance( + new TaskInstanceForceStartRequest(processInstance.getId(), taskGroupQueue.getTaskId())); putMsg(result, Status.SUCCESS); return result; } @@ -1151,19 +1152,9 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ log.error("Process instance does not exist, processInstanceId:{}.", processInstanceId); return null; } - Host host = new Host(processInstance.getHost()); - WorkflowExecutingDataRequest requestCommand = new WorkflowExecutingDataRequest(); - requestCommand.setProcessInstanceId(processInstanceId); - Message message = - stateEventCallbackService.sendSync(host, requestCommand.convert2Command()); - if (message == null) { - log.error("Query executing process instance from master error, processInstanceId:{}.", - processInstanceId); - return null; - } - WorkflowExecutingDataResponse responseCommand = - JSONUtils.parseObject(message.getBody(), WorkflowExecutingDataResponse.class); - return responseCommand.getWorkflowExecuteDto(); + IWorkflowInstanceService iWorkflowInstanceService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(processInstance.getHost(), IWorkflowInstanceService.class); + return iWorkflowInstanceService.getWorkflowExecutingData(processInstanceId); } @Override @@ -1183,9 +1174,9 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ checkMasterExists(); // todo dispatch improvement List masterServerList = monitorService.getServerListFromRegistry(true); - Host host = new Host(masterServerList.get(0).getHost(), masterServerList.get(0).getPort()); + Server server = masterServerList.get(0); - TaskExecuteStartMessage taskExecuteStartMessage = new TaskExecuteStartMessage(); + StreamingTaskTriggerRequest taskExecuteStartMessage = new StreamingTaskTriggerRequest(); taskExecuteStartMessage.setExecutorId(loginUser.getId()); taskExecuteStartMessage.setExecutorName(loginUser.getUserName()); taskExecuteStartMessage.setProjectCode(projectCode); @@ -1198,15 +1189,17 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ taskExecuteStartMessage.setStartParams(startParams); taskExecuteStartMessage.setDryRun(dryRun); - Message response = - stateEventCallbackService.sendSync(host, taskExecuteStartMessage.convert2Command()); - if (response != null) { - log.info("Send task execute start command complete, response is {}.", response); + IStreamingTaskOperator streamingTaskOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(server.getHost() + ":" + server.getPort(), IStreamingTaskOperator.class); + StreamingTaskTriggerResponse streamingTaskTriggerResponse = + streamingTaskOperator.triggerStreamingTask(taskExecuteStartMessage); + if (streamingTaskTriggerResponse.isSuccess()) { + log.info("Send task execute start command complete, response is {}.", streamingTaskOperator); putMsg(result, Status.SUCCESS); } else { log.error( - "Start to execute stream task instance error, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}.", - projectCode, taskDefinitionCode, taskDefinitionVersion); + "Start to execute stream task instance error, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}, response: {}.", + projectCode, taskDefinitionCode, taskDefinitionVersion, streamingTaskTriggerResponse); putMsg(result, Status.START_TASK_INSTANCE_ERROR); } return result; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index ab55a8e1c5..99216e917f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -36,8 +36,18 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.service.log.LogClient; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.IMasterLogService; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.extract.worker.IWorkerLogService; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -67,9 +77,6 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService @Autowired private TaskInstanceDao taskInstanceDao; - @Autowired - private LogClient logClient; - @Autowired ProjectMapper projectMapper; @@ -203,25 +210,46 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService * @return log string data */ private String queryLog(TaskInstance taskInstance, int skipLineNum, int limit) { - Host host = Host.of(taskInstance.getHost()); - String logPath = taskInstance.getLogPath(); - - log.info("Query task instance log, taskInstanceId:{}, taskInstanceName:{}, host:{}, logPath:{}, port:{}", - taskInstance.getId(), taskInstance.getName(), host.getIp(), logPath, host.getPort()); - + final String logPath = taskInstance.getLogPath(); + final String host = taskInstance.getHost(); + log.info("Query task instance log, taskInstanceId:{}, taskInstanceName:{}, host: {}, logPath:{}", + taskInstance.getId(), taskInstance.getName(), taskInstance.getHost(), logPath); StringBuilder sb = new StringBuilder(); if (skipLineNum == 0) { String head = String.format(LOG_HEAD_FORMAT, logPath, - host, + taskInstance.getHost(), Constants.SYSTEM_LINE_SEPARATOR); sb.append(head); } - String logContent = logClient - .rollViewLog(host.getIp(), host.getPort(), logPath, skipLineNum, limit); - - if (skipLineNum == 0 && StringUtils.isEmpty(logContent) && RemoteLogUtils.isRemoteLoggingEnable()) { + String logContent = null; + if (TaskUtils.isLogicTask(taskInstance.getTaskType())) { + IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IMasterLogService.class); + try { + LogicTaskInstanceLogPageQueryRequest logicTaskInstanceLogPageQueryRequest = + new LogicTaskInstanceLogPageQueryRequest(taskInstance.getId(), logPath, skipLineNum, limit); + LogicTaskInstanceLogPageQueryResponse logicTaskInstanceLogPageQueryResponse = + masterLogService.pageQueryLogicTaskInstanceLog(logicTaskInstanceLogPageQueryRequest); + logContent = logicTaskInstanceLogPageQueryResponse.getLogContent(); + } catch (Exception ex) { + log.error("Query LogicTaskInstance log error", ex); + } + } else { + IWorkerLogService iWorkerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(host, IWorkerLogService.class); + try { + TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest = + new TaskInstanceLogPageQueryRequest(taskInstance.getId(), logPath, skipLineNum, limit); + TaskInstanceLogPageQueryResponse taskInstanceLogPageQueryResponse = + iWorkerLogService.pageQueryTaskInstanceLog(taskInstanceLogPageQueryRequest); + logContent = taskInstanceLogPageQueryResponse.getLogContent(); + } catch (Exception ex) { + log.error("Query LogicTaskInstance log error", ex); + } + } + if (logContent == null && RemoteLogUtils.isRemoteLoggingEnable()) { // When getting the log for the first time (skipLineNum=0) returns empty, get the log from remote target try { log.info("Get log {} from remote target", logPath); @@ -233,9 +261,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService log.error("Error while getting log from remote target", e); } } - sb.append(logContent); - return sb.toString(); } @@ -246,7 +272,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService * @return log byte array */ private byte[] getLogBytes(TaskInstance taskInstance) { - Host host = Host.of(taskInstance.getHost()); + String host = taskInstance.getHost(); String logPath = taskInstance.getLogPath(); byte[] head = String.format(LOG_HEAD_FORMAT, @@ -254,9 +280,34 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService host, Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); - byte[] logBytes = logClient.getLogBytes(host.getIp(), host.getPort(), logPath); + byte[] logBytes = new byte[0]; + if (TaskUtils.isLogicTask(taskInstance.getTaskType())) { + IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IMasterLogService.class); + try { + LogicTaskInstanceLogFileDownloadRequest logicTaskInstanceLogFileDownloadRequest = + new LogicTaskInstanceLogFileDownloadRequest(taskInstance.getId(), logPath); + LogicTaskInstanceLogFileDownloadResponse logicTaskInstanceLogFileDownloadResponse = + masterLogService.getLogicTaskInstanceWholeLogFileBytes(logicTaskInstanceLogFileDownloadRequest); + logBytes = logicTaskInstanceLogFileDownloadResponse.getLogBytes(); + } catch (Exception ex) { + log.error("Query LogicTaskInstance log error", ex); + } + } else { + IWorkerLogService iWorkerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(host, IWorkerLogService.class); + try { + TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest = + new TaskInstanceLogFileDownloadRequest(taskInstance.getId(), logPath); + TaskInstanceLogFileDownloadResponse taskInstanceWholeLogFileBytes = + iWorkerLogService.getTaskInstanceWholeLogFileBytes(taskInstanceLogFileDownloadRequest); + logBytes = taskInstanceWholeLogFileBytes.getLogBytes(); + } catch (Exception ex) { + log.error("Query LogicTaskInstance log error", ex); + } + } - if (logBytes.length == 0 && RemoteLogUtils.isRemoteLoggingEnable()) { + if ((logBytes == null || logBytes.length == 0) && RemoteLogUtils.isRemoteLoggingEnable()) { // get task log from remote target try { log.info("Get log {} from remote target", logPath); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MetricsCleanUpServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MetricsCleanUpServiceImpl.java index 70b19ddeac..a7ad669e3b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MetricsCleanUpServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MetricsCleanUpServiceImpl.java @@ -18,13 +18,12 @@ package org.apache.dolphinscheduler.api.service.impl; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; -import org.apache.dolphinscheduler.api.rpc.ApiRpcClient; import org.apache.dolphinscheduler.api.service.MetricsCleanUpService; import org.apache.dolphinscheduler.common.model.Server; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowMetricsCleanUpRequest; -import org.apache.dolphinscheduler.remote.utils.Host; import java.util.List; @@ -37,26 +36,27 @@ import org.springframework.stereotype.Service; @Slf4j public class MetricsCleanUpServiceImpl implements MetricsCleanUpService { - @Autowired - private ApiRpcClient apiRpcClient; - @Autowired private RegistryClient registryClient; @Override - public void cleanUpWorkflowMetricsByDefinitionCode(String workflowDefinitionCode) { - WorkflowMetricsCleanUpRequest workflowMetricsCleanUpRequest = new WorkflowMetricsCleanUpRequest(); - workflowMetricsCleanUpRequest.setProcessDefinitionCode(workflowDefinitionCode); + public void cleanUpWorkflowMetricsByDefinitionCode(Long workflowDefinitionCode) { List masterNodeList = registryClient.getServerList(RegistryNodeType.MASTER); for (Server server : masterNodeList) { - try { - final String host = String.format("%s:%s", server.getHost(), server.getPort()); - apiRpcClient.send(Host.of(host), workflowMetricsCleanUpRequest.convert2Command()); - } catch (Exception e) { - log.error( - "Fail to clean up workflow related metrics on {} when deleting workflow definition {}, error message {}", - server.getHost(), workflowDefinitionCode, e.getMessage()); - } + cleanUpWorkflowMetrics(server, workflowDefinitionCode); + } + } + + private void cleanUpWorkflowMetrics(Server server, Long workflowDefinitionCode) { + try { + IWorkflowInstanceService iWorkflowInstanceService = + SingletonJdkDynamicRpcClientProxyFactory.getInstance().getProxyClient( + String.format("%s:%s", server.getHost(), server.getPort()), IWorkflowInstanceService.class); + iWorkflowInstanceService.clearWorkflowMetrics(workflowDefinitionCode); + } catch (Exception e) { + log.error( + "Fail to clean up workflow related metrics on {} when deleting workflow definition {}, error message {}", + server.getHost(), workflowDefinitionCode, e.getMessage()); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index c26c9bb2c9..d98d7b75fc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -984,7 +984,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro for (ProcessDefinition process : processDefinitionList) { try { this.deleteProcessDefinitionByCode(loginUser, process.getCode()); - metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(process.getCode())); + metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(process.getCode()); } catch (Exception e) { throw new ServiceException(Status.DELETE_PROCESS_DEFINE_ERROR, process.getName(), e.getMessage()); } @@ -1069,7 +1069,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // we delete the workflow definition at last to avoid using transaction here. // If delete error, we can call this interface again. processDefinitionDao.deleteByWorkflowDefinitionCode(processDefinition.getCode()); - metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(code)); + metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(code); log.info("Success delete workflow definition workflowDefinitionCode: {}", code); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index 7551e3a1fa..91156a6890 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -43,12 +43,17 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.repository.DqExecuteResultDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.IMasterLogService; +import org.apache.dolphinscheduler.extract.worker.IStreamingTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.IWorkerLogService; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointResponse; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.task.TaskKillRequest; -import org.apache.dolphinscheduler.remote.command.task.TaskSavePointRequest; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.service.log.LogClient; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskUtils; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.commons.lang3.StringUtils; @@ -98,12 +103,6 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst @Autowired TaskDefinitionMapper taskDefinitionMapper; - @Autowired - private StateEventCallbackService stateEventCallbackService; - - @Autowired - private LogClient logClient; - @Autowired private DqExecuteResultDao dqExecuteResultDao; @@ -290,13 +289,13 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } - - TaskSavePointRequest command = new TaskSavePointRequest(taskInstanceId); - - Host host = new Host(taskInstance.getHost()); - stateEventCallbackService.sendResult(host, command.convert2Command()); + IStreamingTaskInstanceOperator streamingTaskInstanceOperator = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IStreamingTaskInstanceOperator.class); + TaskInstanceTriggerSavepointResponse taskInstanceTriggerSavepointResponse = + streamingTaskInstanceOperator.triggerSavepoint(new TaskInstanceTriggerSavepointRequest(taskInstanceId)); + log.info("StreamingTaskInstance trigger savepoint response: {}", taskInstanceTriggerSavepointResponse); putMsg(result, Status.SUCCESS); - return result; } @@ -322,11 +321,14 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst return result; } - TaskKillRequest command = new TaskKillRequest(taskInstanceId); - Host host = new Host(taskInstance.getHost()); - stateEventCallbackService.sendResult(host, command.convert2Command()); - putMsg(result, Status.SUCCESS); + // todo: we only support streaming task for now + ITaskInstanceOperator iTaskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ITaskInstanceOperator.class); + TaskInstanceKillResponse taskInstanceKillResponse = + iTaskInstanceOperator.killTask(new TaskInstanceKillRequest(taskInstanceId)); + log.info("TaskInstance kill response: {}", taskInstanceKillResponse); + putMsg(result, Status.SUCCESS); return result; } @@ -378,7 +380,15 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst for (TaskInstance taskInstance : needToDeleteTaskInstances) { // delete log if (StringUtils.isNotEmpty(taskInstance.getLogPath())) { - logClient.removeTaskLog(Host.of(taskInstance.getHost()), taskInstance.getLogPath()); + if (TaskUtils.isLogicTask(taskInstance.getTaskType())) { + IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IMasterLogService.class); + masterLogService.removeLogicTaskInstanceLog(taskInstance.getLogPath()); + } else { + IWorkerLogService workerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IWorkerLogService.class); + workerLogService.removeTaskInstanceLog(taskInstance.getLogPath()); + } } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java index 8243cbff51..e08e8cc797 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.k8s; import org.apache.dolphinscheduler.dao.entity.Cluster; import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import java.util.ArrayList; import java.util.List; @@ -57,7 +56,7 @@ public class K8sManagerTest { } @Test - public void getK8sClient() throws RemotingException { + public void getK8sClient() { Mockito.when(clusterMapper.selectList(Mockito.any())).thenReturn(getClusterList()); KubernetesClient result = k8sManager.getK8sClient(1L); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ClusterServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ClusterServiceTest.java index ea1f930b2d..deb04f4199 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ClusterServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ClusterServiceTest.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.Cluster; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.commons.collections4.CollectionUtils; @@ -122,7 +121,7 @@ public class ClusterServiceTest { } @Test - public void testUpdateClusterByCode() throws RemotingException { + public void testUpdateClusterByCode() { User loginUser = getGeneralUser(); Map result = clusterService.updateClusterByCode(loginUser, 1L, clusterName, getConfig(), getDesc()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java index ff2aad42cc..d7a65115de 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java @@ -67,7 +67,6 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.TriggerRelationService; @@ -161,9 +160,6 @@ public class ExecuteFunctionServiceTest { @Mock private ProcessDefinitionService processDefinitionService; - @Mock - private StateEventCallbackService stateEventCallbackService; - private int processDefinitionId = 1; private int processDefinitionVersion = 1; @@ -258,13 +254,6 @@ public class ExecuteFunctionServiceTest { .thenReturn(processTaskRelations); } - @Test - public void testForceStartTaskInstance() { - - Map result = executorService.forceStartTaskInstance(loginUser, taskQueueId); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - } - /** * not complement */ diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 8b0594faf6..0cabfdebd1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -33,7 +33,6 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; -import org.apache.dolphinscheduler.service.log.LogClient; import java.text.MessageFormat; import java.util.HashMap; @@ -75,9 +74,6 @@ public class LoggerServiceTest { @Mock private TaskDefinitionMapper taskDefinitionMapper; - @Mock - private LogClient logClient; - @Test public void testQueryLog() { @@ -134,6 +130,7 @@ public class LoggerServiceTest { User loginUser = new User(); loginUser.setId(1); TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1); taskInstance.setExecutorId(loginUser.getId() + 1); Mockito.when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); @@ -179,11 +176,9 @@ public class LoggerServiceTest { // SUCCESS Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(loginUser, project, DOWNLOAD_LOG); - Mockito.when(logClient.getLogBytes(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString())) - .thenReturn(new byte[0]); Mockito.when(projectMapper.queryProjectByTaskInstanceId(1)).thenReturn(project); byte[] result = loggerService.getLogBytes(loginUser, 1); - Assertions.assertEquals(62, result.length); + Assertions.assertEquals(47, result.length); } @Test @@ -238,8 +233,6 @@ public class LoggerServiceTest { .thenReturn(result); Mockito.when(taskInstanceDao.queryById(1)).thenReturn(taskInstance); Mockito.when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition); - Mockito.when(logClient.getLogBytes(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString())) - .thenReturn(new byte[0]); loggerService.getLogBytes(loginUser, projectCode, 1); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 62d5496d05..705642a88d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -472,7 +472,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool { @Test public void deleteProcessDefinitionByCodeTest() { Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode)); - Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(46L)); + Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(46L); Project project = getProject(projectCode); @@ -520,7 +520,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool { Mockito.when(workFlowLineageService.queryTaskDepOnProcess(project.getCode(), processDefinition.getCode())) .thenReturn(Collections.emptySet()); processDefinitionService.deleteProcessDefinitionByCode(user, 46L); - Mockito.verify(metricsCleanUpService, times(1)).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(46L)); + Mockito.verify(metricsCleanUpService, times(1)).cleanUpWorkflowMetricsByDefinitionCode(46L); // scheduler online Schedule schedule = getSchedule(); @@ -547,7 +547,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool { Mockito.when(workFlowLineageService.queryTaskDepOnProcess(project.getCode(), processDefinition.getCode())) .thenReturn(Collections.emptySet()); Assertions.assertDoesNotThrow(() -> processDefinitionService.deleteProcessDefinitionByCode(user, 46L)); - Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(46L)); + Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(46L); } @Test @@ -596,11 +596,11 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool { Mockito.when(workFlowLineageService.queryTaskDepOnProcess(project.getCode(), process.getCode())) .thenReturn(Collections.emptySet()); putMsg(result, Status.SUCCESS, projectCode); - Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(11L)); + Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(11L); Map deleteSuccess = processDefinitionService.batchDeleteProcessDefinitionByCodes(user, projectCode, singleCodes); Assertions.assertEquals(Status.SUCCESS, deleteSuccess.get(Constants.STATUS)); - Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(11L)); + Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(11L); } @Test diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 0254e6c43a..44f8ab65ad 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -140,6 +140,16 @@ system-lambda + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + com.google.cloud google-cloud-storage diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index ab26aeda4a..53e23d938a 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -222,7 +222,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. avro 1.7.7: https://github.com/apache/avro, Apache 2.0 bonecp 0.8.0.RELEASE: https://github.com/wwadge/bonecp, Apache 2.0 bucket4j-core 6.2.0: https://mvnrepository.com/artifact/com.github.vladimir-bukhtoyarov/bucket4j-core, Apache 2.0 - byte-buddy 1.9.16: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.16, Apache 2.0 caffeine 2.9.3: https://mvnrepository.com/artifact/com.github.ben-manes.caffeine/caffeine/2.9.3, Apache 2.0 classmate 1.5.1: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1, Apache 2.0 clickhouse-jdbc 0.4.6: https://mvnrepository.com/artifact/com.clickhouse/clickhouse-jdbc/0.4.6, Apache 2.0 @@ -408,10 +407,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt. woodstox-core 5.3.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/5.3.0, Apache 2.0 zookeeper 3.8.0: https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper/3.8.0, Apache 2.0 presto-jdbc 0.238.1 https://mvnrepository.com/artifact/com.facebook.presto/presto-jdbc/0.238.1 - protostuff-core 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 - protostuff-runtime 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 - protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0 - protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 prometheus client_java(simpleclient) 0.15.0: https://github.com/prometheus/client_java, Apache 2.0 snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0 trino-jdbc 402: https://mvnrepository.com/artifact/io.trino/trino-jdbc/402, Apache 2.0 @@ -712,12 +707,6 @@ The following components are provided under a MPL 1.1 license. See project link The text of each license is also included at licenses/LICENSE-[project].txt. jamon-runtime 2.3.1: https://mvnrepository.com/artifact/org.jamon/jamon-runtime/2.3.1, MPL-1.1 - javassist 3.27.0-GA: https://github.com/jboss-javassist/javassist, MPL-1.1 - -======================================== -WTFPL License -======================================== - reflections 0.9.12: https://github.com/ronmamo/reflections WTFPL ======================================== CC0-1.0 licenses diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index 1c1a226d5b..3228e3ff2c 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -515,34 +515,6 @@ The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- -============================================================== - protostuff - Copyright 2009 David Yu dyuproject@gmail.com -============================================================== - -protobuf is copyright Google inc unless otherwise noted. -It is licensed under the BSD license. - -jackson-core-asl is copyright FasterXml unless otherwise noted. -It is licensed under the apache 2.0 license. - -antlr is copyright Terence Parr unless otherwise noted. -It is licensed under the BSD license. - -stringtemplate is copyright Terence Parr unless otherwise noted. -It is licensed under the BSD license. - -velocity is licensed under the apache 2.0 license. - -B64Code.java is copyright Mort Bay Consulting Pty Ltd unless otherwise noted. -It is licensed under the apache 2.0 license. - -jarjar is copyright Google inc unless otherwise noted. -It is licensed under the apache 2.0 license. - -guava is copyright Google inc unless otherwise noted. -It is licensed under the apache 2.0 license. - This product includes parquet-tools, initially developed at ARRIS, Inc. with the following copyright notice: diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-byte-buddy.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-byte-buddy.txt deleted file mode 100644 index 8f71f43fee..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-byte-buddy.txt +++ /dev/null @@ -1,202 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt deleted file mode 100644 index 8b7c1537ba..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt +++ /dev/null @@ -1,469 +0,0 @@ - MOZILLA PUBLIC LICENSE - Version 1.1 - - --------------- - -1. Definitions. - - 1.0.1. "Commercial Use" means distribution or otherwise making the - Covered Code available to a third party. - - 1.1. "Contributor" means each entity that creates or contributes to - the creation of Modifications. - - 1.2. "Contributor Version" means the combination of the Original - Code, prior Modifications used by a Contributor, and the Modifications - made by that particular Contributor. - - 1.3. "Covered Code" means the Original Code or Modifications or the - combination of the Original Code and Modifications, in each case - including portions thereof. - - 1.4. "Electronic Distribution Mechanism" means a mechanism generally - accepted in the software development community for the electronic - transfer of data. - - 1.5. "Executable" means Covered Code in any form other than Source - Code. - - 1.6. "Initial Developer" means the individual or entity identified - as the Initial Developer in the Source Code notice required by Exhibit - A. - - 1.7. "Larger Work" means a work which combines Covered Code or - portions thereof with code not governed by the terms of this License. - - 1.8. "License" means this document. - - 1.8.1. "Licensable" means having the right to grant, to the maximum - extent possible, whether at the time of the initial grant or - subsequently acquired, any and all of the rights conveyed herein. - - 1.9. "Modifications" means any addition to or deletion from the - substance or structure of either the Original Code or any previous - Modifications. When Covered Code is released as a series of files, a - Modification is: - A. Any addition to or deletion from the contents of a file - containing Original Code or previous Modifications. - - B. Any new file that contains any part of the Original Code or - previous Modifications. - - 1.10. "Original Code" means Source Code of computer software code - which is described in the Source Code notice required by Exhibit A as - Original Code, and which, at the time of its release under this - License is not already Covered Code governed by this License. - - 1.10.1. "Patent Claims" means any patent claim(s), now owned or - hereafter acquired, including without limitation, method, process, - and apparatus claims, in any patent Licensable by grantor. - - 1.11. "Source Code" means the preferred form of the Covered Code for - making modifications to it, including all modules it contains, plus - any associated interface definition files, scripts used to control - compilation and installation of an Executable, or source code - differential comparisons against either the Original Code or another - well known, available Covered Code of the Contributor's choice. The - Source Code can be in a compressed or archival form, provided the - appropriate decompression or de-archiving software is widely available - for no charge. - - 1.12. "You" (or "Your") means an individual or a legal entity - exercising rights under, and complying with all of the terms of, this - License or a future version of this License issued under Section 6.1. - For legal entities, "You" includes any entity which controls, is - controlled by, or is under common control with You. For purposes of - this definition, "control" means (a) the power, direct or indirect, - to cause the direction or management of such entity, whether by - contract or otherwise, or (b) ownership of more than fifty percent - (50%) of the outstanding shares or beneficial ownership of such - entity. - -2. Source Code License. - - 2.1. The Initial Developer Grant. - The Initial Developer hereby grants You a world-wide, royalty-free, - non-exclusive license, subject to third party intellectual property - claims: - (a) under intellectual property rights (other than patent or - trademark) Licensable by Initial Developer to use, reproduce, - modify, display, perform, sublicense and distribute the Original - Code (or portions thereof) with or without Modifications, and/or - as part of a Larger Work; and - - (b) under Patents Claims infringed by the making, using or - selling of Original Code, to make, have made, use, practice, - sell, and offer for sale, and/or otherwise dispose of the - Original Code (or portions thereof). - - (c) the licenses granted in this Section 2.1(a) and (b) are - effective on the date Initial Developer first distributes - Original Code under the terms of this License. - - (d) Notwithstanding Section 2.1(b) above, no patent license is - granted: 1) for code that You delete from the Original Code; 2) - separate from the Original Code; or 3) for infringements caused - by: i) the modification of the Original Code or ii) the - combination of the Original Code with other software or devices. - - 2.2. Contributor Grant. - Subject to third party intellectual property claims, each Contributor - hereby grants You a world-wide, royalty-free, non-exclusive license - - (a) under intellectual property rights (other than patent or - trademark) Licensable by Contributor, to use, reproduce, modify, - display, perform, sublicense and distribute the Modifications - created by such Contributor (or portions thereof) either on an - unmodified basis, with other Modifications, as Covered Code - and/or as part of a Larger Work; and - - (b) under Patent Claims infringed by the making, using, or - selling of Modifications made by that Contributor either alone - and/or in combination with its Contributor Version (or portions - of such combination), to make, use, sell, offer for sale, have - made, and/or otherwise dispose of: 1) Modifications made by that - Contributor (or portions thereof); and 2) the combination of - Modifications made by that Contributor with its Contributor - Version (or portions of such combination). - - (c) the licenses granted in Sections 2.2(a) and 2.2(b) are - effective on the date Contributor first makes Commercial Use of - the Covered Code. - - (d) Notwithstanding Section 2.2(b) above, no patent license is - granted: 1) for any code that Contributor has deleted from the - Contributor Version; 2) separate from the Contributor Version; - 3) for infringements caused by: i) third party modifications of - Contributor Version or ii) the combination of Modifications made - by that Contributor with other software (except as part of the - Contributor Version) or other devices; or 4) under Patent Claims - infringed by Covered Code in the absence of Modifications made by - that Contributor. - -3. Distribution Obligations. - - 3.1. Application of License. - The Modifications which You create or to which You contribute are - governed by the terms of this License, including without limitation - Section 2.2. The Source Code version of Covered Code may be - distributed only under the terms of this License or a future version - of this License released under Section 6.1, and You must include a - copy of this License with every copy of the Source Code You - distribute. You may not offer or impose any terms on any Source Code - version that alters or restricts the applicable version of this - License or the recipients' rights hereunder. However, You may include - an additional document offering the additional rights described in - Section 3.5. - - 3.2. Availability of Source Code. - Any Modification which You create or to which You contribute must be - made available in Source Code form under the terms of this License - either on the same media as an Executable version or via an accepted - Electronic Distribution Mechanism to anyone to whom you made an - Executable version available; and if made available via Electronic - Distribution Mechanism, must remain available for at least twelve (12) - months after the date it initially became available, or at least six - (6) months after a subsequent version of that particular Modification - has been made available to such recipients. You are responsible for - ensuring that the Source Code version remains available even if the - Electronic Distribution Mechanism is maintained by a third party. - - 3.3. Description of Modifications. - You must cause all Covered Code to which You contribute to contain a - file documenting the changes You made to create that Covered Code and - the date of any change. You must include a prominent statement that - the Modification is derived, directly or indirectly, from Original - Code provided by the Initial Developer and including the name of the - Initial Developer in (a) the Source Code, and (b) in any notice in an - Executable version or related documentation in which You describe the - origin or ownership of the Covered Code. - - 3.4. Intellectual Property Matters - (a) Third Party Claims. - If Contributor has knowledge that a license under a third party's - intellectual property rights is required to exercise the rights - granted by such Contributor under Sections 2.1 or 2.2, - Contributor must include a text file with the Source Code - distribution titled "LEGAL" which describes the claim and the - party making the claim in sufficient detail that a recipient will - know whom to contact. If Contributor obtains such knowledge after - the Modification is made available as described in Section 3.2, - Contributor shall promptly modify the LEGAL file in all copies - Contributor makes available thereafter and shall take other steps - (such as notifying appropriate mailing lists or newsgroups) - reasonably calculated to inform those who received the Covered - Code that new knowledge has been obtained. - - (b) Contributor APIs. - If Contributor's Modifications include an application programming - interface and Contributor has knowledge of patent licenses which - are reasonably necessary to implement that API, Contributor must - also include this information in the LEGAL file. - - (c) Representations. - Contributor represents that, except as disclosed pursuant to - Section 3.4(a) above, Contributor believes that Contributor's - Modifications are Contributor's original creation(s) and/or - Contributor has sufficient rights to grant the rights conveyed by - this License. - - 3.5. Required Notices. - You must duplicate the notice in Exhibit A in each file of the Source - Code. If it is not possible to put such notice in a particular Source - Code file due to its structure, then You must include such notice in a - location (such as a relevant directory) where a user would be likely - to look for such a notice. If You created one or more Modification(s) - You may add your name as a Contributor to the notice described in - Exhibit A. You must also duplicate this License in any documentation - for the Source Code where You describe recipients' rights or ownership - rights relating to Covered Code. You may choose to offer, and to - charge a fee for, warranty, support, indemnity or liability - obligations to one or more recipients of Covered Code. However, You - may do so only on Your own behalf, and not on behalf of the Initial - Developer or any Contributor. You must make it absolutely clear than - any such warranty, support, indemnity or liability obligation is - offered by You alone, and You hereby agree to indemnify the Initial - Developer and every Contributor for any liability incurred by the - Initial Developer or such Contributor as a result of warranty, - support, indemnity or liability terms You offer. - - 3.6. Distribution of Executable Versions. - You may distribute Covered Code in Executable form only if the - requirements of Section 3.1-3.5 have been met for that Covered Code, - and if You include a notice stating that the Source Code version of - the Covered Code is available under the terms of this License, - including a description of how and where You have fulfilled the - obligations of Section 3.2. The notice must be conspicuously included - in any notice in an Executable version, related documentation or - collateral in which You describe recipients' rights relating to the - Covered Code. You may distribute the Executable version of Covered - Code or ownership rights under a license of Your choice, which may - contain terms different from this License, provided that You are in - compliance with the terms of this License and that the license for the - Executable version does not attempt to limit or alter the recipient's - rights in the Source Code version from the rights set forth in this - License. If You distribute the Executable version under a different - license You must make it absolutely clear that any terms which differ - from this License are offered by You alone, not by the Initial - Developer or any Contributor. You hereby agree to indemnify the - Initial Developer and every Contributor for any liability incurred by - the Initial Developer or such Contributor as a result of any such - terms You offer. - - 3.7. Larger Works. - You may create a Larger Work by combining Covered Code with other code - not governed by the terms of this License and distribute the Larger - Work as a single product. In such a case, You must make sure the - requirements of this License are fulfilled for the Covered Code. - -4. Inability to Comply Due to Statute or Regulation. - - If it is impossible for You to comply with any of the terms of this - License with respect to some or all of the Covered Code due to - statute, judicial order, or regulation then You must: (a) comply with - the terms of this License to the maximum extent possible; and (b) - describe the limitations and the code they affect. Such description - must be included in the LEGAL file described in Section 3.4 and must - be included with all distributions of the Source Code. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Application of this License. - - This License applies to code to which the Initial Developer has - attached the notice in Exhibit A and to related Covered Code. - -6. Versions of the License. - - 6.1. New Versions. - Netscape Communications Corporation ("Netscape") may publish revised - and/or new versions of the License from time to time. Each version - will be given a distinguishing version number. - - 6.2. Effect of New Versions. - Once Covered Code has been published under a particular version of the - License, You may always continue to use it under the terms of that - version. You may also choose to use such Covered Code under the terms - of any subsequent version of the License published by Netscape. No one - other than Netscape has the right to modify the terms applicable to - Covered Code created under this License. - - 6.3. Derivative Works. - If You create or use a modified version of this License (which you may - only do in order to apply it to code which is not already Covered Code - governed by this License), You must (a) rename Your license so that - the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", - "MPL", "NPL" or any confusingly similar phrase do not appear in your - license (except to note that your license differs from this License) - and (b) otherwise make it clear that Your version of the license - contains terms which differ from the Mozilla Public License and - Netscape Public License. (Filling in the name of the Initial - Developer, Original Code or Contributor in the notice described in - Exhibit A shall not of themselves be deemed to be modifications of - this License.) - -7. DISCLAIMER OF WARRANTY. - - COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, - WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, - WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF - DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. - THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE - IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, - YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE - COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER - OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF - ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -8. TERMINATION. - - 8.1. This License and the rights granted hereunder will terminate - automatically if You fail to comply with terms herein and fail to cure - such breach within 30 days of becoming aware of the breach. All - sublicenses to the Covered Code which are properly granted shall - survive any termination of this License. Provisions which, by their - nature, must remain in effect beyond the termination of this License - shall survive. - - 8.2. If You initiate litigation by asserting a patent infringement - claim (excluding declatory judgment actions) against Initial Developer - or a Contributor (the Initial Developer or Contributor against whom - You file such action is referred to as "Participant") alleging that: - - (a) such Participant's Contributor Version directly or indirectly - infringes any patent, then any and all rights granted by such - Participant to You under Sections 2.1 and/or 2.2 of this License - shall, upon 60 days notice from Participant terminate prospectively, - unless if within 60 days after receipt of notice You either: (i) - agree in writing to pay Participant a mutually agreeable reasonable - royalty for Your past and future use of Modifications made by such - Participant, or (ii) withdraw Your litigation claim with respect to - the Contributor Version against such Participant. If within 60 days - of notice, a reasonable royalty and payment arrangement are not - mutually agreed upon in writing by the parties or the litigation claim - is not withdrawn, the rights granted by Participant to You under - Sections 2.1 and/or 2.2 automatically terminate at the expiration of - the 60 day notice period specified above. - - (b) any software, hardware, or device, other than such Participant's - Contributor Version, directly or indirectly infringes any patent, then - any rights granted to You by such Participant under Sections 2.1(b) - and 2.2(b) are revoked effective as of the date You first made, used, - sold, distributed, or had made, Modifications made by that - Participant. - - 8.3. If You assert a patent infringement claim against Participant - alleging that such Participant's Contributor Version directly or - indirectly infringes any patent where such claim is resolved (such as - by license or settlement) prior to the initiation of patent - infringement litigation, then the reasonable value of the licenses - granted by such Participant under Sections 2.1 or 2.2 shall be taken - into account in determining the amount or value of any payment or - license. - - 8.4. In the event of termination under Sections 8.1 or 8.2 above, - all end user license agreements (excluding distributors and resellers) - which have been validly granted by You or any distributor hereunder - prior to termination shall survive termination. - -9. LIMITATION OF LIABILITY. - - UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT - (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL - DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, - OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR - ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY - CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, - WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER - COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN - INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF - LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY - RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW - PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE - EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO - THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -10. U.S. GOVERNMENT END USERS. - - The Covered Code is a "commercial item," as that term is defined in - 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer - software" and "commercial computer software documentation," as such - terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 - C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), - all U.S. Government End Users acquire Covered Code with only those - rights set forth herein. - -11. MISCELLANEOUS. - - This License represents the complete agreement concerning subject - matter hereof. If any provision of this License is held to be - unenforceable, such provision shall be reformed only to the extent - necessary to make it enforceable. This License shall be governed by - California law provisions (except to the extent applicable law, if - any, provides otherwise), excluding its conflict-of-law provisions. - With respect to disputes in which at least one party is a citizen of, - or an entity chartered or registered to do business in the United - States of America, any litigation relating to this License shall be - subject to the jurisdiction of the Federal Courts of the Northern - District of California, with venue lying in Santa Clara County, - California, with the losing party responsible for costs, including - without limitation, court costs and reasonable attorneys' fees and - expenses. The application of the United Nations Convention on - Contracts for the International Sale of Goods is expressly excluded. - Any law or regulation which provides that the language of a contract - shall be construed against the drafter shall not apply to this - License. - -12. RESPONSIBILITY FOR CLAIMS. - - As between Initial Developer and the Contributors, each party is - responsible for claims and damages arising, directly or indirectly, - out of its utilization of rights under this License and You agree to - work with Initial Developer and Contributors to distribute such - responsibility on an equitable basis. Nothing herein is intended or - shall be deemed to constitute any admission of liability. - -13. MULTIPLE-LICENSED CODE. - - Initial Developer may designate portions of the Covered Code as - "Multiple-Licensed". "Multiple-Licensed" means that the Initial - Developer permits you to utilize portions of the Covered Code under - Your choice of the MPL or the alternative licenses, if any, specified - by the Initial Developer in the file described in Exhibit A. - -EXHIBIT A -Mozilla Public License. - - ``The contents of this file are subject to the Mozilla Public License - Version 1.1 (the "License"); you may not use this file except in - compliance with the License. You may obtain a copy of the License at - https://www.mozilla.org/MPL/ - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the - License for the specific language governing rights and limitations - under the License. - - The Original Code is ______________________________________. - - The Initial Developer of the Original Code is ________________________. - Portions created by ______________________ are Copyright (C) ______ - _______________________. All Rights Reserved. - - Contributor(s): ______________________________________. - - Alternatively, the contents of this file may be used under the terms - of the _____ license (the "[___] License"), in which case the - provisions of [______] License are applicable instead of those - above. If you wish to allow use of your version of this file only - under the terms of the [____] License and not to allow others to use - your version of this file under the MPL, indicate your decision by - deleting the provisions above and replace them with the notice and - other provisions required by the [___] License. If you do not delete - the provisions above, a recipient may use your version of this file - under either the MPL or the [___] License." - - [NOTE: The text of this Exhibit A may differ slightly from the text of - the notices in the Source Code files of the Original Code. You should - use the text of this Exhibit A rather than the text found in the - Original Code Source Code for Your Modifications.] \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt deleted file mode 100644 index c3155d2478..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt +++ /dev/null @@ -1,13 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-alert/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-alert/pom.xml new file mode 100644 index 0000000000..1ba94d750d --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-extract + dev-SNAPSHOT + + + dolphinscheduler-extract-alert + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-base + + + + diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/IAlertOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/IAlertOperator.java new file mode 100644 index 0000000000..295bd3bd62 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/IAlertOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.alert; + +import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; + +@RpcService +public interface IAlertOperator { + + @RpcMethod + AlertSendResponse sendAlert(AlertSendRequest alertSendRequest); + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java similarity index 73% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java index 68e29f4c28..dfa6515dde 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java @@ -15,10 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.alert; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.alert.request; import lombok.AllArgsConstructor; import lombok.Data; @@ -27,7 +24,7 @@ import lombok.NoArgsConstructor; @Data @NoArgsConstructor @AllArgsConstructor -public class AlertSendRequest implements RequestMessageBuilder { +public class AlertSendRequest { private int groupId; @@ -37,8 +34,4 @@ public class AlertSendRequest implements RequestMessageBuilder { private int warnType; - @Override - public MessageType getCommandType() { - return MessageType.ALERT_SEND_REQUEST; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java similarity index 79% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java index 0f07a5c240..832f3e1fab 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java @@ -15,10 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.alert; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; +package org.apache.dolphinscheduler.extract.alert.request; import java.io.Serializable; import java.util.List; @@ -30,7 +27,7 @@ import lombok.NoArgsConstructor; @Data @NoArgsConstructor @AllArgsConstructor -public class AlertSendResponse implements ResponseMessageBuilder { +public class AlertSendResponse { /** * true:All alert are successful, @@ -40,11 +37,6 @@ public class AlertSendResponse implements ResponseMessageBuilder { private List resResults; - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } - @Data @NoArgsConstructor @AllArgsConstructor @@ -55,4 +47,5 @@ public class AlertSendResponse implements ResponseMessageBuilder { private String message; } + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml new file mode 100644 index 0000000000..9501d55706 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml @@ -0,0 +1,69 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-extract + dev-SNAPSHOT + + + dolphinscheduler-extract-base + + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + + + + org.apache.dolphinscheduler + dolphinscheduler-common + ${project.version} + + + io.netty + netty-all + + + org.slf4j + slf4j-api + + + com.google.guava + guava + + + org.springframework + spring-context + + + + + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/MasterRpcProcessor.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcRequest.java similarity index 85% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/MasterRpcProcessor.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcRequest.java index 2bac00ab84..7fca89b0ee 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/MasterRpcProcessor.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcRequest.java @@ -15,7 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.processor; +package org.apache.dolphinscheduler.extract.base; + +import java.io.Serializable; + +public interface IRpcRequest extends Serializable { -public interface MasterRpcProcessor extends NettyRequestProcessor { } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/BaseLogProcessor.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcResponse.java similarity index 81% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/BaseLogProcessor.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcResponse.java index c651c01939..498efb1047 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/BaseLogProcessor.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcResponse.java @@ -15,11 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.processor; +package org.apache.dolphinscheduler.extract.base; -import lombok.extern.slf4j.Slf4j; +public interface IRpcResponse { -@Slf4j -public abstract class BaseLogProcessor implements MasterRpcProcessor, WorkerRpcProcessor { + boolean isSuccess(); + + String getMessage(); + + byte[] getBody(); + + byte[] toBytes(); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java new file mode 100644 index 0000000000..2b49b2fec2 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base; + +import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; +import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter; +import org.apache.dolphinscheduler.extract.base.protocal.Transporter; +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; +import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils; + +import java.util.concurrent.ExecutorService; + +import lombok.extern.slf4j.Slf4j; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.timeout.IdleStateEvent; + +@ChannelHandler.Sharable +@Slf4j +public class NettyClientHandler extends ChannelInboundHandlerAdapter { + + private final NettyRemotingClient nettyRemotingClient; + + private final ExecutorService callbackExecutor; + + public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) { + this.nettyRemotingClient = nettyRemotingClient; + this.callbackExecutor = callbackExecutor; + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); + ctx.channel().close(); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + processReceived((Transporter) msg); + } + + private void processReceived(final Transporter transporter) { + ResponseFuture future = ResponseFuture.getFuture(transporter.getHeader().getOpaque()); + if (future == null) { + log.warn("Cannot find the ResponseFuture if transporter: {}", transporter); + return; + } + StandardRpcResponse deserialize = JsonSerializer.deserialize(transporter.getBody(), StandardRpcResponse.class); + future.setIRpcResponse(deserialize); + future.release(); + if (future.getInvokeCallback() != null) { + future.removeFuture(); + this.callbackExecutor.submit(future::executeInvokeCallback); + } else { + future.putResponse(deserialize); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + log.error("NettyClientHandler catch an exception : {}", cause.getMessage(), cause); + nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); + ctx.channel().close(); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + ctx.channel() + .writeAndFlush(HeartBeatTransporter.getHeartBeatTransporter()) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE); + if (log.isDebugEnabled()) { + log.debug("Client send heart beat to: {}", ChannelUtils.getRemoteAddress(ctx.channel())); + } + } else { + super.userEventTriggered(ctx, evt); + } + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java index bbc5f960de..1ca87091af 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java @@ -15,25 +15,23 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; - -import org.apache.dolphinscheduler.remote.codec.NettyDecoder; -import org.apache.dolphinscheduler.remote.codec.NettyEncoder; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.exceptions.RemotingTimeoutException; -import org.apache.dolphinscheduler.remote.exceptions.RemotingTooMuchRequestException; -import org.apache.dolphinscheduler.remote.future.InvokeCallback; -import org.apache.dolphinscheduler.remote.future.ReleaseSemaphore; -import org.apache.dolphinscheduler.remote.future.ResponseFuture; -import org.apache.dolphinscheduler.remote.handler.NettyClientHandler; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.CallerThreadExecutePolicy; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; -import org.apache.dolphinscheduler.remote.utils.NettyUtils; +package org.apache.dolphinscheduler.extract.base; + +import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; +import org.apache.dolphinscheduler.extract.base.exception.RemotingException; +import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException; +import org.apache.dolphinscheduler.extract.base.exception.RemotingTooMuchRequestException; +import org.apache.dolphinscheduler.extract.base.future.InvokeCallback; +import org.apache.dolphinscheduler.extract.base.future.ReleaseSemaphore; +import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; +import org.apache.dolphinscheduler.extract.base.protocal.Transporter; +import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; +import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; +import org.apache.dolphinscheduler.extract.base.utils.CallerThreadExecutePolicy; +import org.apache.dolphinscheduler.extract.base.utils.Constants; +import org.apache.dolphinscheduler.extract.base.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.NamedThreadFactory; +import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; @@ -64,8 +62,6 @@ public class NettyRemotingClient implements AutoCloseable { private final Bootstrap bootstrap = new Bootstrap(); - private final NettyEncoder encoder = new NettyEncoder(); - private final ConcurrentHashMap channels = new ConcurrentHashMap<>(128); private final AtomicBoolean isStarted = new AtomicBoolean(false); @@ -123,24 +119,20 @@ public class NettyRemotingClient implements AutoCloseable { public void initChannel(SocketChannel ch) { ch.pipeline() .addLast("client-idle-handler", - new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, + new IdleStateHandler( + Constants.NETTY_CLIENT_HEART_BEAT_TIME, + 0, + 0, TimeUnit.MILLISECONDS)) - .addLast(new NettyDecoder(), clientHandler, encoder); + .addLast(new TransporterDecoder(), clientHandler, new TransporterEncoder()); } }); this.responseFutureExecutor.scheduleWithFixedDelay(ResponseFuture::scanFutureTable, 0, 1, TimeUnit.SECONDS); isStarted.compareAndSet(false, true); } - /** - * async send - * - * @param host host - * @param command command - * @param timeoutMillis timeoutMillis - * @param invokeCallback callback function - */ - public void sendAsync(final Host host, final Message command, + public void sendAsync(final Host host, + final Transporter transporter, final long timeoutMillis, final InvokeCallback invokeCallback) throws InterruptedException, RemotingException { final Channel channel = getChannel(host); @@ -150,7 +142,7 @@ public class NettyRemotingClient implements AutoCloseable { /* * request unique identification */ - final long opaque = command.getOpaque(); + final long opaque = transporter.getHeader().getOpaque(); /* * control concurrency number */ @@ -166,7 +158,7 @@ public class NettyRemotingClient implements AutoCloseable { invokeCallback, releaseSemaphore); try { - channel.writeAndFlush(command).addListener(future -> { + channel.writeAndFlush(transporter).addListener(future -> { if (future.isSuccess()) { responseFuture.setSendOk(true); return; @@ -185,7 +177,7 @@ public class NettyRemotingClient implements AutoCloseable { }); } catch (Exception ex) { responseFuture.release(); - throw new RemotingException(String.format("send command to host: %s failed", host), ex); + throw new RemotingException(String.format("Send transporter to host: %s failed", host), ex); } } else { String message = String.format( @@ -195,23 +187,15 @@ public class NettyRemotingClient implements AutoCloseable { } } - /** - * sync send - * - * @param host host - * @param message command - * @param timeoutMillis timeoutMillis - * @return command - */ - public Message sendSync(final Host host, final Message message, - final long timeoutMillis) throws InterruptedException, RemotingException { + public IRpcResponse sendSync(final Host host, final Transporter transporter, + final long timeoutMillis) throws InterruptedException, RemotingException { final Channel channel = getChannel(host); if (channel == null) { throw new RemotingException(String.format("connect to : %s fail", host)); } - final long opaque = message.getOpaque(); + final long opaque = transporter.getHeader().getOpaque(); final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null); - channel.writeAndFlush(message).addListener(future -> { + channel.writeAndFlush(transporter).addListener(future -> { if (future.isSuccess()) { responseFuture.setSendOk(true); return; @@ -220,74 +204,22 @@ public class NettyRemotingClient implements AutoCloseable { } responseFuture.setCause(future.cause()); responseFuture.putResponse(null); - log.error("send command {} to host {} failed: {}", message, host, responseFuture.getCause()); + log.error("Send Sync request {} to host {} failed", transporter, host, responseFuture.getCause()); }); /* * sync wait for result */ - Message result = responseFuture.waitResponse(); - if (result == null) { + IRpcResponse iRpcResponse = responseFuture.waitResponse(); + if (iRpcResponse == null) { if (responseFuture.isSendOK()) { throw new RemotingTimeoutException(host.toString(), timeoutMillis, responseFuture.getCause()); } else { throw new RemotingException(host.toString(), responseFuture.getCause()); } } - return result; - } - - /** - * send task - * - * @param host host - * @param message command - */ - public void send(final Host host, final Message message) throws RemotingException { - Channel channel = getChannel(host); - if (channel == null) { - throw new RemotingException(String.format("connect to : %s fail", host)); - } - try { - ChannelFuture future = channel.writeAndFlush(message).await(); - if (future.isSuccess()) { - log.debug("send command : {} , to : {} successfully.", message, host.getAddress()); - } else { - String msg = String.format("send command : %s , to :%s failed", message, host.getAddress()); - log.error(msg, future.cause()); - throw new RemotingException(msg); - } - } catch (RemotingException remotingException) { - throw remotingException; - } catch (Exception e) { - log.error("Send command {} to address {} encounter error.", message, host.getAddress()); - throw new RemotingException( - String.format("Send command : %s , to :%s encounter error", message, host.getAddress()), e); - } - } - - /** - * register processor - * - * @param processor processor - */ - public void registerProcessor(final NettyRequestProcessor processor) { - this.registerProcessor(processor, null); + return iRpcResponse; } - /** - * register processor - * - * @param processor processor - * @param executor thread executor - */ - public void registerProcessor(final NettyRequestProcessor processor, - final ExecutorService executor) { - this.clientHandler.registerProcessor(processor.getCommandType(), processor, executor); - } - - /** - * get channel - */ public Channel getChannel(Host host) { Channel channel = channels.get(host); if (channel != null && channel.isActive()) { @@ -299,7 +231,7 @@ public class NettyRemotingClient implements AutoCloseable { /** * create channel * - * @param host host + * @param host host * @param isSync sync flag * @return channel */ @@ -344,9 +276,6 @@ public class NettyRemotingClient implements AutoCloseable { } } - /** - * close channels - */ private void closeChannels() { for (Channel channel : this.channels.values()) { channel.close(); @@ -354,11 +283,6 @@ public class NettyRemotingClient implements AutoCloseable { this.channels.clear(); } - /** - * close channel - * - * @param host host - */ public void closeChannel(Host host) { Channel channel = this.channels.remove(host); if (channel != null) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingClientFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java similarity index 71% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingClientFactory.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java index 2364f97e06..7bbebfbf3d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingClientFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.factory; +package org.apache.dolphinscheduler.extract.base; -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -27,9 +26,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class NettyRemotingClientFactory { - public NettyRemotingClient buildNettyRemotingClient() { - NettyClientConfig nettyClientConfig = new NettyClientConfig(); - log.info("NettyRemotingClient initialized with config: {}", nettyClientConfig); + public NettyRemotingClient buildNettyRemotingClient(NettyClientConfig nettyClientConfig) { return new NettyRemotingClient(nettyClientConfig); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java similarity index 74% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java index 740910a9a5..50d82e2c91 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; +package org.apache.dolphinscheduler.extract.base; -import org.apache.dolphinscheduler.remote.codec.NettyDecoder; -import org.apache.dolphinscheduler.remote.codec.NettyEncoder; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.exceptions.RemoteException; -import org.apache.dolphinscheduler.remote.handler.NettyServerHandler; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.NettyUtils; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.exception.RemoteException; +import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; +import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; +import org.apache.dolphinscheduler.extract.base.server.JdkDynamicServerHandler; +import org.apache.dolphinscheduler.extract.base.server.ServerMethodInvoker; +import org.apache.dolphinscheduler.extract.base.utils.Constants; +import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -53,51 +53,23 @@ import io.netty.handler.timeout.IdleStateHandler; @Slf4j public class NettyRemotingServer { - /** - * server bootstrap - */ private final ServerBootstrap serverBootstrap = new ServerBootstrap(); - /** - * default executor - */ - private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS); + private final ExecutorService defaultExecutor = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2); - /** - * boss group - */ private final EventLoopGroup bossGroup; - /** - * worker group - */ private final EventLoopGroup workGroup; - /** - * server config - */ private final NettyServerConfig serverConfig; - /** - * server handler - */ - private final NettyServerHandler serverHandler = new NettyServerHandler(this); + private final JdkDynamicServerHandler serverHandler = new JdkDynamicServerHandler(this); - /** - * started flag - */ private final AtomicBoolean isStarted = new AtomicBoolean(false); - /** - * Netty server bind fail message - */ private static final String NETTY_BIND_FAILURE_MSG = "NettyRemotingServer bind %s fail"; - /** - * server init - * - * @param serverConfig server config - */ public NettyRemotingServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; ThreadFactory bossThreadFactory = @@ -113,9 +85,6 @@ public class NettyRemotingServer { } } - /** - * server start - */ public void start() { if (isStarted.compareAndSet(false, true)) { this.serverBootstrap @@ -160,42 +129,21 @@ public class NettyRemotingServer { */ private void initNettyChannel(SocketChannel ch) { ch.pipeline() - .addLast("encoder", new NettyEncoder()) - .addLast("decoder", new NettyDecoder()) + .addLast("encoder", new TransporterEncoder()) + .addLast("decoder", new TransporterDecoder()) .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) .addLast("handler", serverHandler); } - /** - * register processor - * - * @param processor processor - */ - public void registerProcessor(final NettyRequestProcessor processor) { - this.registerProcessor(processor, null); - } - - /** - * register processor - * - * @param processor processor - * @param executor thread executor - */ - public void registerProcessor(final NettyRequestProcessor processor, - final ExecutorService executor) { - this.serverHandler.registerProcessor(processor.getCommandType(), processor, executor); - } - - /** - * get default thread executor - * - * @return thread executor - */ public ExecutorService getDefaultExecutor() { return defaultExecutor; } + public void registerMethodInvoker(ServerMethodInvoker methodInvoker) { + serverHandler.registerMethodInvoker(methodInvoker); + } + public void close() { if (isStarted.compareAndSet(true, false)) { try { diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java similarity index 68% rename from dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java index ad09a34645..6bf1b8d31c 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java @@ -15,23 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.microbench.common; +package org.apache.dolphinscheduler.extract.base; -import org.apache.dolphinscheduler.rpc.base.RpcService; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; -/** - * UserService - */ -@RpcService("IUserService") -public class UserService implements IUserService { +import lombok.experimental.UtilityClass; - @Override - public Boolean say(String s) { - return true; - } +@UtilityClass +public class NettyRemotingServerFactory { - @Override - public Integer hi(int num) { - return ++num; + public NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) { + return new NettyRemotingServer(nettyServerConfig); } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java new file mode 100644 index 0000000000..cd1b778c9a --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface RpcMethod { + + long timeout() default 3000L; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcService.java similarity index 85% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcService.java index 81b16280e0..f4dc931efd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcService.java @@ -15,16 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.base; +package org.apache.dolphinscheduler.extract.base; +import java.lang.annotation.Documented; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -@Target({ElementType.TYPE}) +/** + * Used to mark a class as a RPC service + */ +@Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) +@Documented public @interface RpcService { - - String value() default ""; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcRequest.java similarity index 55% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcRequest.java index 2438c3dc58..f3f0d9fb3a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcRequest.java @@ -15,40 +15,34 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.log; +package org.apache.dolphinscheduler.extract.base; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * roll view log request command - */ @Data -@NoArgsConstructor @AllArgsConstructor -public class RollViewLogRequest implements RequestMessageBuilder { - - /** - * log path - */ - private String path; - - /** - * skip line number - */ - private int skipLineNum; - - /** - * query line number - */ - private int limit; - - @Override - public MessageType getCommandType() { - return MessageType.ROLL_VIEW_LOG_REQUEST; +@NoArgsConstructor +public class StandardRpcRequest implements IRpcRequest { + + private byte[][] args; + + private Class[] argsTypes; + + public static StandardRpcRequest of(Object[] args) { + if (args == null || args.length == 0) { + return new StandardRpcRequest(null, null); + } + final byte[][] argsBytes = new byte[args.length][]; + final Class[] argsTypes = new Class[args.length]; + for (int i = 0; i < args.length; i++) { + argsBytes[i] = JsonSerializer.serialize(args[i]); + argsTypes[i] = args[i].getClass(); + } + return new StandardRpcRequest(argsBytes, argsTypes); } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcResponse.java similarity index 50% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcResponse.java index f9c9bfb319..21a6afa76e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcResponse.java @@ -15,36 +15,40 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; +package org.apache.dolphinscheduler.extract.base; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; -import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; @Data -@NoArgsConstructor -@AllArgsConstructor -public class TaskDispatchResponse implements ResponseMessageBuilder { +public class StandardRpcResponse implements IRpcResponse { - private Integer taskInstanceId; - - private boolean dispatchSuccess; + private boolean success; private String message; - public static TaskDispatchResponse success(Integer taskInstanceId) { - return new TaskDispatchResponse(taskInstanceId, true, "dispatch success"); + private byte[] body; + + private Class bodyType; + + public static StandardRpcResponse success(byte[] body, Class bodyType) { + StandardRpcResponse rpcResponse = new StandardRpcResponse(); + rpcResponse.setSuccess(true); + rpcResponse.setBody(body); + rpcResponse.setBodyType(bodyType); + return rpcResponse; } - public static TaskDispatchResponse failed(Integer taskInstanceId, String message) { - return new TaskDispatchResponse(taskInstanceId, false, message); + public static StandardRpcResponse fail(String message) { + StandardRpcResponse rpcResponse = new StandardRpcResponse(); + rpcResponse.setSuccess(false); + rpcResponse.setMessage(message); + return rpcResponse; } @Override - public MessageType getCommandType() { - return MessageType.TASK_DISPATCH_RESPONSE; + public byte[] toBytes() { + return JsonSerializer.serialize(this); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java similarity index 50% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java index 57cec4bbe7..519dd87199 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java @@ -15,42 +15,28 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.client; -/** - * key value pair - * - * @param L generic type - * @param R generic type - */ -public class Pair { +import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; +import org.apache.dolphinscheduler.extract.base.utils.Host; - private L left; +import java.lang.reflect.Method; - private R right; +public abstract class BaseRemoteMethodInvoker implements ClientMethodInvoker { - public Pair(L left, R right) { - this.left = left; - this.right = right; - } + protected final String methodIdentifier; - public L getLeft() { - return left; - } + protected final NettyRemotingClient nettyRemotingClient; - public void setLeft(L left) { - this.left = left; - } + protected final Method localMethod; - public R getRight() { - return right; - } + protected final Host serverHost; - public void setRight(R right) { - this.right = right; + public BaseRemoteMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { + this.serverHost = serverHost; + this.localMethod = localMethod; + this.nettyRemotingClient = nettyRemotingClient; + this.methodIdentifier = localMethod.toGenericString(); } - public static Pair of(L left, R right) { - return new Pair<>(left, right); - } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java new file mode 100644 index 0000000000..d5c9ab73d3 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.client; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.utils.Host; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ClientInvocationHandler implements InvocationHandler { + + private final NettyRemotingClient nettyRemotingClient; + + private final Map methodInvokerMap; + + private final Host serverHost; + + public ClientInvocationHandler(Host serverHost, NettyRemotingClient nettyRemotingClient) { + this.serverHost = checkNotNull(serverHost); + this.nettyRemotingClient = checkNotNull(nettyRemotingClient); + this.methodInvokerMap = new ConcurrentHashMap<>(); + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (method.getAnnotation(RpcMethod.class) == null) { + return method.invoke(proxy, args); + } + ClientMethodInvoker methodInvoker = methodInvokerMap.computeIfAbsent( + method.toGenericString(), m -> new SyncClientMethodInvoker(serverHost, method, nettyRemotingClient)); + return methodInvoker.invoke(proxy, method, args); + } + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java new file mode 100644 index 0000000000..dcf53b0311 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.client; + +import java.lang.reflect.Method; + +public interface ClientMethodInvoker { + + Object invoke(Object proxy, Method method, Object[] args) throws Throwable; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java similarity index 68% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java index 2b106a96e4..e60b0f18b0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.common; +package org.apache.dolphinscheduler.extract.base.client; -/** - * AbstractRpcCallBack - */ -public abstract class AbstractRpcCallBack { +public interface IRpcClientProxyFactory { /** - * When sending an asynchronous message, this method will be called after the response is successfully sent. + * Create the client proxy. * - * @param object response + * @param serverHost which server host to connect + * @param clientInterface which need to be proxied + * @param InterfaceType + * @return proxy client */ - public abstract void run(Object object); + T getProxyClient(String serverHost, Class clientInterface); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java new file mode 100644 index 0000000000..25e6e32ee4 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.client; + +import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; +import org.apache.dolphinscheduler.extract.base.utils.Host; + +import java.lang.reflect.Proxy; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * This class is used to create a proxy client which will transform local method invocation to remove invocation. + */ +public class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory { + + private final NettyRemotingClient nettyRemotingClient; + + // todo: use guava cache to avoid memory leak + private final Map> proxyClientCache = new ConcurrentHashMap<>(); + + public JdkDynamicRpcClientProxyFactory(NettyRemotingClient nettyRemotingClient) { + this.nettyRemotingClient = nettyRemotingClient; + } + + @SuppressWarnings("unchecked") + @Override + public T getProxyClient(String serverHost, Class clientInterface) { + return (T) proxyClientCache + .computeIfAbsent(serverHost, key -> new ConcurrentHashMap<>()) + .computeIfAbsent(clientInterface.getName(), + key -> Proxy.newProxyInstance( + clientInterface.getClassLoader(), new Class[]{clientInterface}, + new ClientInvocationHandler(Host.of(serverHost), nettyRemotingClient))); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskPauseRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java similarity index 54% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskPauseRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java index 210a5155bb..44e0420272 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskPauseRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java @@ -15,31 +15,22 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; +package org.apache.dolphinscheduler.extract.base.client; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +import org.apache.dolphinscheduler.extract.base.NettyRemotingClientFactory; +import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; +public class SingletonJdkDynamicRpcClientProxyFactory extends JdkDynamicRpcClientProxyFactory { -@Data -@NoArgsConstructor -@ToString(callSuper = true) -public class TaskPauseRequest implements RequestMessageBuilder { + private static final SingletonJdkDynamicRpcClientProxyFactory INSTANCE = + new SingletonJdkDynamicRpcClientProxyFactory(); - private static final long serialVersionUID = -1L; - - private Integer taskInstanceId; - - public TaskPauseRequest(Integer taskInstanceId) { - this.taskInstanceId = taskInstanceId; + private SingletonJdkDynamicRpcClientProxyFactory() { + super(NettyRemotingClientFactory.buildNettyRemotingClient(new NettyClientConfig())); } - @Override - public MessageType getCommandType() { - return MessageType.PAUSE_TASK_INSTANCE; + public static SingletonJdkDynamicRpcClientProxyFactory getInstance() { + return INSTANCE; } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java new file mode 100644 index 0000000000..b5fdf3fb71 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.client; + +import org.apache.dolphinscheduler.extract.base.IRpcResponse; +import org.apache.dolphinscheduler.extract.base.NettyRemotingClient; +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; +import org.apache.dolphinscheduler.extract.base.exception.MethodInvocationException; +import org.apache.dolphinscheduler.extract.base.protocal.Transporter; +import org.apache.dolphinscheduler.extract.base.protocal.TransporterHeader; +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; +import org.apache.dolphinscheduler.extract.base.utils.Host; + +import java.lang.reflect.Method; + +public class SyncClientMethodInvoker extends BaseRemoteMethodInvoker { + + public SyncClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) { + super(serverHost, localMethod, nettyRemotingClient); + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + RpcMethod sync = method.getAnnotation(RpcMethod.class); + Transporter transporter = new Transporter(); + transporter.setBody(JsonSerializer.serialize(StandardRpcRequest.of(args))); + transporter.setHeader(TransporterHeader.of(methodIdentifier)); + + IRpcResponse iRpcResponse = + nettyRemotingClient.sendSync(serverHost, transporter, sync.timeout()); + if (!iRpcResponse.isSuccess()) { + throw MethodInvocationException.of(iRpcResponse.getMessage()); + } + if (iRpcResponse.getBody() == null) { + return null; + } + Class responseClass = method.getReturnType(); + return JsonSerializer.deserialize(iRpcResponse.getBody(), responseClass); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java similarity index 91% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java index ff41e9b101..a41a439b5c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.config; - -import org.apache.dolphinscheduler.remote.utils.Constants; +package org.apache.dolphinscheduler.extract.base.config; import lombok.AllArgsConstructor; import lombok.Builder; @@ -34,7 +32,7 @@ public class NettyClientConfig { * worker threads，default get machine cpus */ @Builder.Default - private int workerThreads = Constants.CPUS; + private int workerThreads = Runtime.getRuntime().availableProcessors() * 2; /** * whether tpc delay diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java similarity index 75% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java index 798ea732c5..f81804935b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.remote.config; -import org.apache.dolphinscheduler.remote.utils.Constants; +package org.apache.dolphinscheduler.extract.base.config; import lombok.Data; @@ -29,32 +28,36 @@ public class NettyServerConfig { private int soBacklog = 1024; /** - * whether tpc delay + * whether tpc delay */ private boolean tcpNoDelay = true; /** - * whether keep alive + * whether keep alive */ private boolean soKeepalive = true; /** - * send buffer size + * send buffer size */ private int sendBufferSize = 65535; /** - * receive buffer size + * receive buffer size */ private int receiveBufferSize = 65535; /** - * worker threads，default get machine cpus + * worker threads，default get machine cpus */ - private int workerThread = Constants.CPUS; + private int workerThread = Runtime.getRuntime().availableProcessors() * 2; /** - * listen port + * listen port */ private int listenPort = 12346; + + public NettyServerConfig(int listenPort) { + this.listenPort = listenPort; + } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/MethodInvocationException.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/MethodInvocationException.java new file mode 100644 index 0000000000..1c20b5fd2d --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/MethodInvocationException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.exception; + +public class MethodInvocationException extends RuntimeException { + + public MethodInvocationException(String message) { + super(message); + } + + public static MethodInvocationException of(String message) { + return new MethodInvocationException(message); + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemoteException.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemoteException.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemoteException.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemoteException.java index a13859026d..94f83796ae 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemoteException.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemoteException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.exceptions; +package org.apache.dolphinscheduler.extract.base.exception; /** * Custom runtime exception diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingException.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingException.java index 1d025cc268..4290362a01 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.exceptions; +package org.apache.dolphinscheduler.extract.base.exception; /** * remote exception diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTimeoutException.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTimeoutException.java index 5788221d6c..22b77bf1f5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTimeoutException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.exceptions; +package org.apache.dolphinscheduler.extract.base.exception; /** * timeout exception diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTooMuchRequestException.java similarity index 94% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTooMuchRequestException.java index 4e31497311..fc82298acd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTooMuchRequestException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.remote.exceptions; +package org.apache.dolphinscheduler.extract.base.exception; /** * too much request exception diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/InvokeCallback.java similarity index 94% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/InvokeCallback.java index 84cdae867b..4df3d68e2f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/InvokeCallback.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.remote.future; +package org.apache.dolphinscheduler.extract.base.future; /** * invoke callback diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ReleaseSemaphore.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ReleaseSemaphore.java index c8284890f9..e5f8d5ab98 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ReleaseSemaphore.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.remote.future; +package org.apache.dolphinscheduler.extract.base.future; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java similarity index 73% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java index 40b223d6e4..35405c5578 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.future; +package org.apache.dolphinscheduler.extract.base.future; -import org.apache.dolphinscheduler.remote.command.Message; +import org.apache.dolphinscheduler.extract.base.IRpcResponse; import java.util.Iterator; import java.util.Map; @@ -25,50 +25,41 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; -/** - * response future - */ +@ToString @Slf4j public class ResponseFuture { private static final ConcurrentHashMap FUTURE_TABLE = new ConcurrentHashMap<>(256); - /** - * request unique identification - */ private final long opaque; - /** - * timeout - */ + // remove the timeout private final long timeoutMillis; - /** - * invokeCallback function - */ private final InvokeCallback invokeCallback; - /** - * releaseSemaphore - */ private final ReleaseSemaphore releaseSemaphore; private final CountDownLatch latch = new CountDownLatch(1); private final long beginTimestamp = System.currentTimeMillis(); - /** - * response command - */ - private Message responseMessage; + @Getter + @Setter + private IRpcResponse iRpcResponse; private volatile boolean sendOk = true; private Throwable cause; - public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, + public ResponseFuture(long opaque, + long timeoutMillis, + InvokeCallback invokeCallback, ReleaseSemaphore releaseSemaphore) { this.opaque = opaque; this.timeoutMillis = timeoutMillis; @@ -82,20 +73,15 @@ public class ResponseFuture { * * @return command */ - public Message waitResponse() throws InterruptedException { + public IRpcResponse waitResponse() throws InterruptedException { if (!latch.await(timeoutMillis, TimeUnit.MILLISECONDS)) { - log.warn("Wait response timeout, request id {}", opaque); + log.warn("Wait response in {}/ms timeout, request id {}", timeoutMillis, opaque); } - return this.responseMessage; + return this.iRpcResponse; } - /** - * put response - * - * @param responseMessage responseCommand - */ - public void putResponse(final Message responseMessage) { - this.responseMessage = responseMessage; + public void putResponse(final IRpcResponse iRpcResponse) { + this.iRpcResponse = iRpcResponse; this.latch.countDown(); FUTURE_TABLE.remove(opaque); } @@ -155,14 +141,6 @@ public class ResponseFuture { return beginTimestamp; } - public Message getResponseCommand() { - return responseMessage; - } - - public void setResponseCommand(Message responseMessage) { - this.responseMessage = responseMessage; - } - public InvokeCallback getInvokeCallback() { return invokeCallback; } @@ -199,18 +177,4 @@ public class ResponseFuture { } } - @Override - public String toString() { - return "ResponseFuture{" - + "opaque=" + opaque - + ", timeoutMillis=" + timeoutMillis - + ", invokeCallback=" + invokeCallback - + ", releaseSemaphore=" + releaseSemaphore - + ", latch=" + latch - + ", beginTimestamp=" + beginTimestamp - + ", responseCommand=" + responseMessage - + ", sendOk=" + sendOk - + ", cause=" + cause - + '}'; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/HeartBeatTransporter.java similarity index 62% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/HeartBeatTransporter.java index f559e30faf..dba8b5bc0b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/HeartBeatTransporter.java @@ -15,28 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.common; +package org.apache.dolphinscheduler.extract.base.protocal; -public enum ResponseEventType { +import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; - ACK((byte) 1, "ack"), - BUSINESS_RSP((byte) 2, "business response"); +public class HeartBeatTransporter extends Transporter { - private Byte type; + public static final String METHOD_IDENTIFY = "HEART_BEAT"; - private String description; + private static final Transporter HEART_BEAT_TRANSPORTER = + of(TransporterHeader.of(METHOD_IDENTIFY), new StandardRpcResponse()); - ResponseEventType(Byte type, String description) { - this.type = type; - this.description = description; - } - - public Byte getType() { - return type; - } - - public String getDescription() { - return description; + public static Transporter getHeartBeatTransporter() { + return HEART_BEAT_TRANSPORTER; } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/Transporter.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/Transporter.java new file mode 100644 index 0000000000..073c1485d9 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/Transporter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.protocal; + +import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; +import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; + +import java.io.Serializable; + +import lombok.Data; +import lombok.NonNull; + +@Data +public class Transporter implements Serializable { + + private static final long serialVersionUID = -1L; + + public static final byte MAGIC = (byte) 0xbabe; + public static final byte VERSION = 0; + + private TransporterHeader header; + private byte[] body; + + public static Transporter of(@NonNull TransporterHeader header, StandardRpcResponse iRpcResponse) { + return of(header, JsonSerializer.serialize(iRpcResponse)); + } + + public static Transporter of(@NonNull TransporterHeader header, StandardRpcRequest iRpcRequest) { + return of(header, JsonSerializer.serialize(iRpcRequest)); + } + + public static Transporter of(@NonNull TransporterHeader header, byte[] body) { + Transporter transporter = new Transporter(); + transporter.setHeader(header); + transporter.setBody(body); + return transporter; + } + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterDecoder.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterDecoder.java new file mode 100644 index 0000000000..8f2747c079 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterDecoder.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.protocal; + +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ReplayingDecoder; + +@Slf4j +public class TransporterDecoder extends ReplayingDecoder { + + public TransporterDecoder() { + super(State.MAGIC); + } + + private int headerLength; + private byte[] header; + private int bodyLength; + private byte[] body; + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { + switch (state()) { + case MAGIC: + checkMagic(in.readByte()); + checkpoint(State.VERSION); + case VERSION: + checkVersion(in.readByte()); + checkpoint(State.HEADER_LENGTH); + case HEADER_LENGTH: + headerLength = in.readInt(); + checkpoint(State.HEADER); + case HEADER: + header = new byte[headerLength]; + in.readBytes(header); + checkpoint(State.BODY_LENGTH); + case BODY_LENGTH: + bodyLength = in.readInt(); + checkpoint(State.BODY); + case BODY: + body = new byte[bodyLength]; + in.readBytes(body); + Transporter transporter = + Transporter.of(JsonSerializer.deserialize(header, TransporterHeader.class), body); + out.add(transporter); + checkpoint(State.MAGIC); + break; + default: + log.warn("unknown decoder state {}", state()); + } + } + + private void checkMagic(byte magic) { + if (magic != Transporter.MAGIC) { + throw new IllegalArgumentException("illegal packet [magic]" + magic); + } + } + + private void checkVersion(byte version) { + if (version != Transporter.VERSION) { + throw new IllegalArgumentException("illegal protocol [version]" + version); + } + } + + enum State { + MAGIC, + VERSION, + HEADER_LENGTH, + HEADER, + BODY_LENGTH, + BODY; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterEncoder.java similarity index 51% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterEncoder.java index d6e10cf126..246a313b6c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterEncoder.java @@ -15,46 +15,35 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.codec; +package org.apache.dolphinscheduler.extract.base.protocal; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; +import org.apache.dolphinscheduler.extract.base.exception.RemotingException; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandler.Sharable; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.MessageToByteEncoder; -/** - * netty encoder - */ @Sharable -public class NettyEncoder extends MessageToByteEncoder { - - /** - * encode - * - * @param ctx channel handler context - * @param msg command - * @param out byte buffer - */ +public class TransporterEncoder extends MessageToByteEncoder { + @Override - protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception { - if (msg == null) { + protected void encode(ChannelHandlerContext ctx, Transporter transporter, ByteBuf out) throws Exception { + if (transporter == null) { throw new RemotingException("encode msg is null"); } - out.writeByte(Message.MAGIC); - out.writeByte(Message.VERSION); - out.writeByte(msg.getType().ordinal()); - out.writeLong(msg.getOpaque()); - writeContext(msg, out); - out.writeInt(msg.getBody().length); - out.writeBytes(msg.getBody()); - } + out.writeByte(Transporter.MAGIC); + out.writeByte(Transporter.VERSION); - private void writeContext(Message msg, ByteBuf out) { - byte[] headerBytes = msg.getContext().toBytes(); - out.writeInt(headerBytes.length); - out.writeBytes(headerBytes); + // write header + byte[] header = transporter.getHeader().toBytes(); + out.writeInt(header.length); + out.writeBytes(header); + + // write body + byte[] body = transporter.getBody(); + out.writeInt(body.length); + out.writeBytes(body); } + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterHeader.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterHeader.java new file mode 100644 index 0000000000..abeafd9160 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterHeader.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.protocal; + +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; + +import java.io.Serializable; +import java.util.concurrent.atomic.AtomicLong; + +import lombok.Data; + +@Data +public class TransporterHeader implements Serializable { + + private static final long serialVersionUID = -1L; + + private static final AtomicLong REQUEST_ID = new AtomicLong(1); + + private String methodIdentifier; + private long opaque; + + // Used for JsonDeSerializer + public TransporterHeader() { + + } + + public TransporterHeader(String methodIdentifier) { + this(REQUEST_ID.getAndIncrement(), methodIdentifier); + } + + public TransporterHeader(long opaque, String methodIdentifier) { + this.opaque = opaque; + this.methodIdentifier = methodIdentifier; + } + + public static TransporterHeader of(String methodIdentifier) { + return new TransporterHeader(methodIdentifier); + } + + public static TransporterHeader of(long opaque, String methodIdentifier) { + return new TransporterHeader(opaque, methodIdentifier); + } + + public byte[] toBytes() { + return JsonSerializer.serialize(this); + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java similarity index 91% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java index da3db257a6..56e19fe8b2 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.serialize; import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; @@ -68,14 +68,15 @@ public class JsonSerializer { * @return byte array */ public static byte[] serialize(T obj) { - String json = ""; + if (obj == null) { + return null; + } try { - json = objectMapper.writeValueAsString(obj); + return objectMapper.writeValueAsString(obj).getBytes(StandardCharsets.UTF_8); } catch (JsonProcessingException e) { log.error("serializeToString exception!", e); + return null; } - - return json.getBytes(Constants.UTF8); } /** @@ -99,12 +100,15 @@ public class JsonSerializer { /** * deserialize * - * @param src byte array + * @param src byte array * @param clazz class - * @param deserialize type + * @param deserialize type * @return deserialize type */ public static T deserialize(byte[] src, Class clazz) { + if (src == null) { + return null; + } String json = new String(src, StandardCharsets.UTF_8); try { @@ -113,7 +117,6 @@ public class JsonSerializer { log.error("deserialize exception!", e); return null; } - } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java new file mode 100644 index 0000000000..7e5feab05f --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; +import org.apache.dolphinscheduler.extract.base.StandardRpcRequest; +import org.apache.dolphinscheduler.extract.base.StandardRpcResponse; +import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter; +import org.apache.dolphinscheduler.extract.base.protocal.Transporter; +import org.apache.dolphinscheduler.extract.base.protocal.TransporterHeader; +import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer; +import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; + +import lombok.extern.slf4j.Slf4j; +import io.netty.channel.Channel; +import io.netty.channel.ChannelConfig; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.timeout.IdleStateEvent; + +@Slf4j +@ChannelHandler.Sharable +public class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter { + + private final NettyRemotingServer nettyRemotingServer; + + private final Map methodInvokerMap; + + public JdkDynamicServerHandler(NettyRemotingServer nettyRemotingServer) { + this.nettyRemotingServer = nettyRemotingServer; + this.methodInvokerMap = new ConcurrentHashMap<>(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + ctx.channel().close(); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + processReceived(ctx.channel(), (Transporter) msg); + } + + public void registerMethodInvoker(ServerMethodInvoker methodInvoker) { + checkNotNull(methodInvoker); + checkNotNull(methodInvoker.getMethodIdentify()); + + methodInvokerMap.put(methodInvoker.getMethodIdentify(), methodInvoker); + } + + private void processReceived(final Channel channel, final Transporter transporter) { + final String methodIdentifier = transporter.getHeader().getMethodIdentifier(); + if (HeartBeatTransporter.METHOD_IDENTIFY.equals(methodIdentifier)) { + if (log.isDebugEnabled()) { + log.debug("server receive heart beat from: host: {}", ChannelUtils.getRemoteAddress(channel)); + } + return; + } + ServerMethodInvoker methodInvoker = methodInvokerMap.get(methodIdentifier); + try { + if (methodInvoker == null) { + log.error("Cannot find the ServerMethodInvoker of : {}", transporter); + StandardRpcResponse iRpcResponse = + StandardRpcResponse.fail("Cannot find the ServerMethodInvoker of " + methodIdentifier); + TransporterHeader transporterHeader = + TransporterHeader.of(transporter.getHeader().getOpaque(), methodIdentifier); + Transporter response = Transporter.of(transporterHeader, iRpcResponse); + channel.writeAndFlush(response); + return; + } + nettyRemotingServer.getDefaultExecutor().submit(() -> { + StandardRpcResponse iRpcResponse; + try { + StandardRpcRequest standardRpcRequest = + JsonSerializer.deserialize(transporter.getBody(), StandardRpcRequest.class); + Object[] args; + if (standardRpcRequest.getArgs().length == 0) { + args = null; + } else { + args = new Object[standardRpcRequest.getArgs().length]; + for (int i = 0; i < standardRpcRequest.getArgs().length; i++) { + args[i] = JsonSerializer.deserialize(standardRpcRequest.getArgs()[i], + standardRpcRequest.getArgsTypes()[i]); + } + } + Object result = methodInvoker.invoke(args); + if (result == null) { + iRpcResponse = StandardRpcResponse.success(null, null); + } else { + iRpcResponse = StandardRpcResponse.success(JsonSerializer.serialize(result), result.getClass()); + } + } catch (Throwable e) { + log.error("Invoke method {} failed, {}.", methodIdentifier, e.getMessage(), e); + iRpcResponse = StandardRpcResponse.fail(e.getMessage()); + } + TransporterHeader transporterHeader = + TransporterHeader.of(transporter.getHeader().getOpaque(), methodIdentifier); + Transporter response = Transporter.of(transporterHeader, iRpcResponse); + channel.writeAndFlush(response); + }); + } catch (RejectedExecutionException e) { + log.warn("NettyRemotingServer's thread pool is full, discard msg {} from {}", transporter, + ChannelUtils.getRemoteAddress(channel)); + StandardRpcResponse iRpcResponse = StandardRpcResponse.fail("NettyRemotingServer's thread pool is full"); + TransporterHeader transporterHeader = + TransporterHeader.of(transporter.getHeader().getOpaque(), methodIdentifier); + Transporter response = Transporter.of(transporterHeader, iRpcResponse); + channel.writeAndFlush(response); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + log.error("exceptionCaught : {}", cause.getMessage(), cause); + ctx.channel().close(); + } + + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) { + Channel ch = ctx.channel(); + ChannelConfig config = ch.config(); + + if (!ch.isWritable()) { + if (log.isWarnEnabled()) { + log.warn("{} is not writable, over high water level : {}", + ch, config.getWriteBufferHighWaterMark()); + } + + config.setAutoRead(false); + } else { + if (log.isWarnEnabled()) { + log.warn("{} is writable, to low water : {}", ch, config.getWriteBufferLowWaterMark()); + } + config.setAutoRead(true); + } + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + ctx.channel().close(); + } else { + super.userEventTriggered(ctx, evt); + } + } + +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java similarity index 81% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java index 3d65c308e1..ee633217b2 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.service.cache; +package org.apache.dolphinscheduler.extract.base.server; -import org.apache.dolphinscheduler.remote.command.Message; +public interface ServerMethodInvoker { -public interface CacheNotifyService { + String getMethodIdentify(); + + Object invoke(final Object... arg) throws Throwable; - void notifyMaster(Message message); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java new file mode 100644 index 0000000000..f81b901a5f --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +import java.lang.reflect.Method; + +public class ServerMethodInvokerImpl implements ServerMethodInvoker { + + private final Object serviceBean; + + private final Method method; + + private final String methodIdentify; + + public ServerMethodInvokerImpl(Object serviceBean, Method method) { + this.serviceBean = serviceBean; + this.method = method; + this.methodIdentify = method.toGenericString(); + } + + @Override + public Object invoke(Object... args) throws Throwable { + // todo: check the request param when register + return method.invoke(serviceBean, args); + } + + @Override + public String getMethodIdentify() { + return methodIdentify; + } +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java new file mode 100644 index 0000000000..ea6eb008a4 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.server; + +import org.apache.dolphinscheduler.extract.base.NettyRemotingServer; +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; + +import java.lang.reflect.Method; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.lang.Nullable; + +@Slf4j +public class SpringServerMethodInvokerDiscovery implements BeanPostProcessor { + + protected final NettyRemotingServer nettyRemotingServer; + + public SpringServerMethodInvokerDiscovery(NettyRemotingServer nettyRemotingServer) { + this.nettyRemotingServer = nettyRemotingServer; + } + + @Nullable + @Override + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + Class[] interfaces = bean.getClass().getInterfaces(); + for (Class anInterface : interfaces) { + if (anInterface.getAnnotation(RpcService.class) == null) { + continue; + } + registerRpcMethodInvoker(anInterface, bean, beanName); + } + return bean; + } + + private void registerRpcMethodInvoker(Class anInterface, Object bean, String beanName) { + Method[] declaredMethods = anInterface.getDeclaredMethods(); + for (Method method : declaredMethods) { + RpcMethod rpcMethod = method.getAnnotation(RpcMethod.class); + if (rpcMethod == null) { + continue; + } + ServerMethodInvoker methodInvoker = new ServerMethodInvokerImpl(bean, method); + nettyRemotingServer.registerMethodInvoker(methodInvoker); + log.info("Register ServerMethodInvoker: {} to bean: {}", methodInvoker.getMethodIdentify(), beanName); + } + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/CallerThreadExecutePolicy.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/CallerThreadExecutePolicy.java index 981b1d8c24..bc14b7cb4f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/CallerThreadExecutePolicy.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.utils; import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/ChannelUtils.java similarity index 97% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/ChannelUtils.java index 66f1794fdb..3643d17e39 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/ChannelUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.utils; import org.apache.dolphinscheduler.common.utils.NetUtils; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java index 8611dda54c..76e3872d31 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.utils; import org.apache.dolphinscheduler.common.utils.NetUtils; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Host.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Host.java index 94a852ea33..4f14bc0275 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Host.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.utils; import static org.apache.dolphinscheduler.common.constants.Constants.COLON; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NamedThreadFactory.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NamedThreadFactory.java index 69f69c1ba0..19589a9698 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NamedThreadFactory.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.utils; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NettyUtils.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java rename to dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NettyUtils.java index e91f06dd0c..ceaef9647f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NettyUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.utils; +package org.apache.dolphinscheduler.extract.base.utils; import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollServerSocketChannel; diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml new file mode 100644 index 0000000000..04afc0b1de --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-extract + dev-SNAPSHOT + + + dolphinscheduler-extract-master + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-base + ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + + + + diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java new file mode 100644 index 0000000000..a4fb547028 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest; + +@RpcService +public interface ILogicTaskInstanceOperator { + + @RpcMethod + LogicTaskDispatchResponse dispatchLogicTask(LogicTaskDispatchRequest taskDispatchRequest); + + @RpcMethod + LogicTaskKillResponse killLogicTask(LogicTaskKillRequest taskKillRequest); + + @RpcMethod + LogicTaskPauseResponse pauseLogicTask(LogicTaskPauseRequest taskPauseRequest); + + @RpcMethod + TaskInstanceForceStartResponse forceStartTaskInstance(TaskInstanceForceStartRequest taskForceStartRequest); + + @RpcMethod + void wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterCacheService.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterCacheService.java new file mode 100644 index 0000000000..36746c4d13 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterCacheService.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.master.transportor.CacheExpireRequest; + +@RpcService +public interface IMasterCacheService { + + @RpcMethod + void cacheExpire(CacheExpireRequest cacheExpireRequest); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterLogService.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterLogService.java new file mode 100644 index 0000000000..a7f15b2ea8 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterLogService.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryResponse; + +@RpcService +public interface IMasterLogService { + + @RpcMethod + LogicTaskInstanceLogFileDownloadResponse getLogicTaskInstanceWholeLogFileBytes(LogicTaskInstanceLogFileDownloadRequest logicTaskInstanceLogFileDownloadRequest); + + @RpcMethod + LogicTaskInstanceLogPageQueryResponse pageQueryLogicTaskInstanceLog(LogicTaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest); + + @RpcMethod + void removeLogicTaskInstanceLog(String taskInstanceLogAbsolutePath); +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IStreamingTaskOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IStreamingTaskOperator.java new file mode 100644 index 0000000000..f98e249fbd --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IStreamingTaskOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; + +@RpcService +public interface IStreamingTaskOperator { + + @RpcMethod + StreamingTaskTriggerResponse triggerStreamingTask(StreamingTaskTriggerRequest streamingTaskTriggerRequest); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskInstanceExecutionEventListener.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskInstanceExecutionEventListener.java new file mode 100644 index 0000000000..fefd8a0574 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskInstanceExecutionEventListener.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionInfoEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionRunningEvent; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; + +@RpcService +public interface ITaskInstanceExecutionEventListener { + + @RpcMethod + void onTaskInstanceExecutionRunning(TaskInstanceExecutionRunningEvent taskInstanceExecutionRunningEvent); + + @RpcMethod + void onTaskInstanceExecutionFinish(TaskInstanceExecutionFinishEvent taskInstanceExecutionFinishEvent); + + @RpcMethod + void onTaskInstanceExecutionInfoUpdate(TaskInstanceExecutionInfoEvent taskInstanceExecutionInfoEvent); + + // todo: Split this into a separate interface + @RpcMethod + void onWorkflowInstanceInstanceStateChange(WorkflowInstanceStateChangeEvent workflowInstanceStateChangeEvent); + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowMetricsCleanUpRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java similarity index 63% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowMetricsCleanUpRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java index 4faf09f123..217e81322e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowMetricsCleanUpRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.workflow; +package org.apache.dolphinscheduler.extract.master; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; -import lombok.Data; +@RpcService +public interface IWorkflowInstanceService { -@Data -public class WorkflowMetricsCleanUpRequest implements RequestMessageBuilder { + @RpcMethod + void clearWorkflowMetrics(Long workflowDefinitionCode); - private String processDefinitionCode; + @RpcMethod + WorkflowExecuteDto getWorkflowExecutingData(Integer workflowInstanceId); - @Override - public MessageType getCommandType() { - return MessageType.WORKFLOW_METRICS_CLEANUP; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/TaskInstanceExecuteDto.java similarity index 97% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/TaskInstanceExecuteDto.java index 0bffe9272b..fa24a151eb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/TaskInstanceExecuteDto.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.dto; +package org.apache.dolphinscheduler.extract.master.dto; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/WorkflowExecuteDto.java similarity index 98% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/WorkflowExecuteDto.java index a2b5d54f61..33b5444701 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/WorkflowExecuteDto.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.dto; +package org.apache.dolphinscheduler.extract.master.dto; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/CacheExpireRequest.java similarity index 74% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/CacheExpireRequest.java index 76c63dd87d..fa31242925 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/CacheExpireRequest.java @@ -15,27 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.cache; +package org.apache.dolphinscheduler.extract.master.transportor; import org.apache.dolphinscheduler.common.enums.CacheType; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@NoArgsConstructor @AllArgsConstructor -public class CacheExpireRequest implements RequestMessageBuilder { +@NoArgsConstructor +public class CacheExpireRequest { private CacheType cacheType; private String cacheKey; - @Override - public MessageType getCommandType() { - return MessageType.CACHE_EXPIRE; - } - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskInstanceExecutionEvent.java similarity index 64% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskInstanceExecutionEvent.java index d7023f1c01..3fa5ace8ce 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskInstanceExecutionEvent.java @@ -15,27 +15,26 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.protocol; +package org.apache.dolphinscheduler.extract.master.transportor; -public class RpcProtocol { +public interface ITaskInstanceExecutionEvent { - private MessageHeader msgHeader; + long getEventCreateTime(); - private T body; + void setEventCreateTime(long eventCreateTime); - public MessageHeader getMsgHeader() { - return msgHeader; - } + long getEventSendTime(); - public void setMsgHeader(MessageHeader msgHeader) { - this.msgHeader = msgHeader; - } + void setEventSendTime(long eventSendTime); - public T getBody() { - return body; - } + void setHost(String host); - public void setBody(T body) { - this.body = body; + TaskInstanceExecutionEventType getEventType(); + + enum TaskInstanceExecutionEventType { + RUNNING, + FINISH, + RUNNING_INFO } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchRequest.java similarity index 71% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchRequest.java index 000cce9fc3..660ee23de9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchRequest.java @@ -15,31 +15,22 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; +package org.apache.dolphinscheduler.extract.master.transportor; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; + +import java.io.Serializable; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * The task dispatch message, means dispatch a task to worker. - */ @Data @NoArgsConstructor @AllArgsConstructor -public class TaskDispatchRequest implements RequestMessageBuilder { +public class LogicTaskDispatchRequest implements Serializable { private static final long serialVersionUID = -1L; private TaskExecutionContext taskExecutionContext; - - @Override - public MessageType getCommandType() { - return MessageType.TASK_DISPATCH_REQUEST; - } - } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchResponse.java new file mode 100644 index 0000000000..68ebfba298 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchResponse.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LogicTaskDispatchResponse { + + private Integer taskInstanceId; + + private boolean dispatchSuccess; + + private String message; + + public static LogicTaskDispatchResponse success(Integer taskInstanceId) { + return new LogicTaskDispatchResponse(taskInstanceId, true, "dispatch success"); + } + + public static LogicTaskDispatchResponse failed(Integer taskInstanceId, String message) { + return new LogicTaskDispatchResponse(taskInstanceId, false, message); + } +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadRequest.java new file mode 100644 index 0000000000..7fe4c5df24 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadRequest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LogicTaskInstanceLogFileDownloadRequest { + + private long taskInstanceId; + + private String taskInstanceLogAbsolutePath; +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadResponse.java new file mode 100644 index 0000000000..fe80526e55 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogFileDownloadResponse.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LogicTaskInstanceLogFileDownloadResponse { + + private byte[] logBytes; +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryRequest.java similarity index 73% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryRequest.java index aa9abca9c1..5a8d3da808 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryRequest.java @@ -15,23 +15,21 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command; +package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; -/** - * db task final result response command - */ @Data +@NoArgsConstructor @AllArgsConstructor -public class StateEventResponse implements RequestMessageBuilder { +public class LogicTaskInstanceLogPageQueryRequest { - private String key; + private long taskInstanceId; - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RESULT_MESSAGE_ACK; - } + private String taskInstanceLogAbsolutePath; + private int skipLineNum; + private int limit; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryResponse.java new file mode 100644 index 0000000000..1d4a2b6b55 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskInstanceLogPageQueryResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LogicTaskInstanceLogPageQueryResponse { + + private String logContent; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillRequest.java similarity index 75% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillRequest.java index 609d3b1aed..a1702b588e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillRequest.java @@ -15,15 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.client; +package org.apache.dolphinscheduler.extract.master.transportor; -import org.apache.dolphinscheduler.remote.utils.Host; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; -/** - * IRpcClient - */ -public interface IRpcClient { +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LogicTaskKillRequest { - T create(Class clazz, Host host) throws Exception; + private int taskInstanceId; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillResponse.java similarity index 69% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillResponse.java index b989627f4a..a15b9220ca 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskKillResponse.java @@ -15,26 +15,26 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; - -import java.util.List; +package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@AllArgsConstructor @NoArgsConstructor -public class GetAppIdResponse implements ResponseMessageBuilder { +@AllArgsConstructor +public class LogicTaskKillResponse { - private List appIds; + private boolean success; + private String message; - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; + public static LogicTaskKillResponse success() { + return new LogicTaskKillResponse(true, null); } + + public static LogicTaskKillResponse fail(String message) { + return new LogicTaskKillResponse(false, message); + } + } diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseRequest.java similarity index 75% rename from dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseRequest.java index 130bcf7469..0ac51e0212 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseRequest.java @@ -15,17 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.microbench.common; +package org.apache.dolphinscheduler.extract.master.transportor; -import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; -/** - * UserCallback - */ -public class UserCallback extends AbstractRpcCallBack { +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LogicTaskPauseRequest { - @Override - public void run(Object object) { + private int taskInstanceId; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java index 9366133816..b91c275074 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java @@ -15,30 +15,25 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; +package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * view log response command - */ @Data @NoArgsConstructor @AllArgsConstructor -public class ViewLogResponseResponse implements ResponseMessageBuilder { +public class LogicTaskPauseResponse { - /** - * response data - */ - private String msg; + private boolean success; + private String message; + + public static LogicTaskPauseResponse success() { + return new LogicTaskPauseResponse(true, null); + } - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; + public static LogicTaskPauseResponse fail(String message) { + return new LogicTaskPauseResponse(false, message); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteStartMessage.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java similarity index 63% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteStartMessage.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java index 619fb8e2c9..7f2b16a530 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteStartMessage.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java @@ -15,26 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; +package org.apache.dolphinscheduler.extract.master.transportor; import java.util.Map; import lombok.Data; -import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import lombok.ToString; -/** - * Task execute start message, from api to master. - */ @Data @NoArgsConstructor -@ToString(callSuper = true) -@EqualsAndHashCode(callSuper = true) -public class TaskExecuteStartMessage extends BaseMessage { +public class StreamingTaskTriggerRequest { private int executorId; @@ -58,13 +48,4 @@ public class TaskExecuteStartMessage extends BaseMessage { private int dryRun; - public TaskExecuteStartMessage(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) { - super(messageSenderAddress, messageReceiverAddress, messageSendTime); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_START; - } - } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java new file mode 100644 index 0000000000..0f9f265280 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerResponse.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class StreamingTaskTriggerResponse { + + private boolean success; + + private String message; + + public static StreamingTaskTriggerResponse success() { + return new StreamingTaskTriggerResponse(true, null); + } + + public static StreamingTaskTriggerResponse fail(String message) { + return new StreamingTaskTriggerResponse(false, message); + } + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionFinishEvent.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionFinishEvent.java new file mode 100644 index 0000000000..33ac1280d0 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionFinishEvent.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceExecutionFinishEvent implements ITaskInstanceExecutionEvent { + + private int taskInstanceId; + + private int processInstanceId; + + private int status; + + private long startTime; + + private String host; + + private String logPath; + + private String executePath; + + private long endTime; + + private int processId; + + private String appIds; + + private String varPool; + + private long eventCreateTime; + + private long eventSendTime; + + @Override + public TaskInstanceExecutionEventType getEventType() { + return TaskInstanceExecutionEventType.FINISH; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionInfoEvent.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionInfoEvent.java index e11ba53a8b..22039a9e70 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionInfoEvent.java @@ -15,30 +15,35 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * kill task request command - */ @Data -@NoArgsConstructor @AllArgsConstructor -public class TaskKillRequest implements RequestMessageBuilder { +@NoArgsConstructor +public class TaskInstanceExecutionInfoEvent implements ITaskInstanceExecutionEvent { - /** - * task id - */ private int taskInstanceId; + private int processInstanceId; + + private long startTime; + + private String host; + + private String logPath; + + private int processId; + + private long eventCreateTime; + + private long eventSendTime; + @Override - public MessageType getCommandType() { - return MessageType.TASK_KILL_REQUEST; + public TaskInstanceExecutionEventType getEventType() { + return TaskInstanceExecutionEventType.RUNNING_INFO; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionRunningEvent.java similarity index 68% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionRunningEvent.java index 9ef2bf5312..023d953ad8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskKillResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceExecutionRunningEvent.java @@ -15,43 +15,43 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; +package org.apache.dolphinscheduler.extract.master.transportor; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; - -import java.util.List; import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -/** - * kill task response command - */ @Data -@Builder @NoArgsConstructor @AllArgsConstructor -public class TaskKillResponse implements ResponseMessageBuilder { +public class TaskInstanceExecutionRunningEvent implements ITaskInstanceExecutionEvent { private int taskInstanceId; + private int processInstanceId; + + private long startTime; + private String host; private TaskExecutionStatus status; + private String logPath; + + private String executePath; + private int processId; - /** - * other resource manager appId , for example : YARN etc - */ - private List appIds; + private String appIds; + + private long eventCreateTime; + + private long eventSendTime; @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; + public TaskInstanceExecutionEventType getEventType() { + return TaskInstanceExecutionEventType.RUNNING; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskWakeupRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskWakeupRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java index 3a21d5b32f..0a4711fa1e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskWakeupRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java @@ -15,19 +15,14 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; +package org.apache.dolphinscheduler.extract.master.transportor; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; - -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data @NoArgsConstructor -@AllArgsConstructor -public class TaskWakeupRequest implements RequestMessageBuilder { +public class TaskInstanceForceStartRequest { private String key; @@ -35,17 +30,13 @@ public class TaskWakeupRequest implements RequestMessageBuilder { private int taskInstanceId; - public TaskWakeupRequest( - int processInstanceId, - int taskInstanceId) { + public TaskInstanceForceStartRequest( + int processInstanceId, + int taskInstanceId) { this.key = String.format("%d-%d", processInstanceId, taskInstanceId); this.processInstanceId = processInstanceId; this.taskInstanceId = taskInstanceId; } - @Override - public MessageType getCommandType() { - return MessageType.TASK_WAKEUP_EVENT_REQUEST; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java index 4a1c48c900..db19c023dc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java @@ -15,30 +15,27 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * view log request command - */ @Data @NoArgsConstructor @AllArgsConstructor -public class ViewLogRequest implements RequestMessageBuilder { +public class TaskInstanceForceStartResponse { + + private boolean success; - /** - * log path - */ - private String path; + private String message; - @Override - public MessageType getCommandType() { - return MessageType.VIEW_WHOLE_LOG_REQUEST; + public static TaskInstanceForceStartResponse success() { + return new TaskInstanceForceStartResponse(true, "dispatch success"); } + + public static TaskInstanceForceStartResponse failed(String message) { + return new TaskInstanceForceStartResponse(false, message); + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskForceStartRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java similarity index 66% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskForceStartRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java index 56499065db..5bfbc03f1c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskForceStartRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java @@ -15,19 +15,14 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; +package org.apache.dolphinscheduler.extract.master.transportor; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; - -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data @NoArgsConstructor -@AllArgsConstructor -public class TaskForceStartRequest implements RequestMessageBuilder { +public class TaskInstanceWakeupRequest { private String key; @@ -35,17 +30,13 @@ public class TaskForceStartRequest implements RequestMessageBuilder { private int taskInstanceId; - public TaskForceStartRequest( - int processInstanceId, - int taskInstanceId) { + public TaskInstanceWakeupRequest( + int processInstanceId, + int taskInstanceId) { this.key = String.format("%d-%d", processInstanceId, taskInstanceId); this.processInstanceId = processInstanceId; this.taskInstanceId = taskInstanceId; } - @Override - public MessageType getCommandType() { - return MessageType.TASK_FORCE_STATE_EVENT_REQUEST; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupResponse.java similarity index 68% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupResponse.java index 50b23d40b3..1d8884f7fe 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupResponse.java @@ -15,26 +15,27 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.master.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@AllArgsConstructor @NoArgsConstructor -public class WorkflowHostChangeRequest implements RequestMessageBuilder { +@AllArgsConstructor +public class TaskInstanceWakeupResponse { - private int taskInstanceId; + private boolean success; - private String workflowHost; + private String message; - @Override - public MessageType getCommandType() { - return MessageType.WORKFLOW_HOST_CHANGE_REQUEST; + public static TaskInstanceWakeupResponse success() { + return new TaskInstanceWakeupResponse(true, null); } + + public static TaskInstanceWakeupResponse failed(String message) { + return new TaskInstanceWakeupResponse(false, message); + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowStateEventChangeRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/WorkflowInstanceStateChangeEvent.java similarity index 68% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowStateEventChangeRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/WorkflowInstanceStateChangeEvent.java index b8b771c84e..9064b94fae 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowStateEventChangeRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/WorkflowInstanceStateChangeEvent.java @@ -15,21 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.workflow; +package org.apache.dolphinscheduler.extract.master.transportor; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * db task final result response command - */ @Data +@AllArgsConstructor @NoArgsConstructor -public class WorkflowStateEventChangeRequest implements RequestMessageBuilder { +public class WorkflowInstanceStateChangeEvent { private String key; @@ -43,11 +40,11 @@ public class WorkflowStateEventChangeRequest implements RequestMessageBuilder { private int destTaskInstanceId; - public WorkflowStateEventChangeRequest(int sourceProcessInstanceId, - int sourceTaskInstanceId, - WorkflowExecutionStatus sourceStatus, - int destProcessInstanceId, - int destTaskInstanceId) { + public WorkflowInstanceStateChangeEvent(int sourceProcessInstanceId, + int sourceTaskInstanceId, + WorkflowExecutionStatus sourceStatus, + int destProcessInstanceId, + int destTaskInstanceId) { this.key = String.format("%d-%d-%d-%d", sourceProcessInstanceId, sourceTaskInstanceId, @@ -60,9 +57,4 @@ public class WorkflowStateEventChangeRequest implements RequestMessageBuilder { this.destProcessInstanceId = destProcessInstanceId; this.destTaskInstanceId = destTaskInstanceId; } - - @Override - public MessageType getCommandType() { - return MessageType.STATE_EVENT_REQUEST; - } } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml new file mode 100644 index 0000000000..da5fbd3b3f --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-extract + dev-SNAPSHOT + + + dolphinscheduler-extract-worker + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-base + + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + + + + + diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IStreamingTaskInstanceOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IStreamingTaskInstanceOperator.java new file mode 100644 index 0000000000..00c6b88ef5 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IStreamingTaskInstanceOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointResponse; + +@RpcService +public interface IStreamingTaskInstanceOperator { + + @RpcMethod + TaskInstanceTriggerSavepointResponse triggerSavepoint(TaskInstanceTriggerSavepointRequest taskInstanceTriggerSavepointRequest); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java new file mode 100644 index 0000000000..940d2e2dd1 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionFinishEventAck; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionInfoEventAck; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck; + +@RpcService +public interface ITaskInstanceExecutionEventAckListener { + + // todo: If we use sync, then we don't need ack here + @RpcMethod + void handleTaskInstanceExecutionRunningEventAck(TaskInstanceExecutionRunningEventAck taskInstanceExecutionRunningEventAck); + + @RpcMethod + void handleTaskInstanceExecutionFinishEventAck(TaskInstanceExecutionFinishEventAck taskInstanceExecutionFinishEventAck); + + @RpcMethod + void handleTaskInstanceExecutionInfoEventAck(TaskInstanceExecutionInfoEventAck taskInstanceExecutionInfoEventAck); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java new file mode 100644 index 0000000000..ca80afa8a2 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; + +@RpcService +public interface ITaskInstanceOperator { + + @RpcMethod + TaskInstanceDispatchResponse dispatchTask(TaskInstanceDispatchRequest taskInstanceDispatchRequest); + + @RpcMethod + TaskInstanceKillResponse killTask(TaskInstanceKillRequest taskInstanceKillRequest); + + @RpcMethod + TaskInstancePauseResponse pauseTask(TaskInstancePauseRequest taskPauseRequest); + + @RpcMethod + UpdateWorkflowHostResponse updateWorkflowInstanceHost(UpdateWorkflowHostRequest updateWorkflowHostRequest); + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IWorkerLogService.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IWorkerLogService.java new file mode 100644 index 0000000000..d03191888a --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IWorkerLogService.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker; + +import org.apache.dolphinscheduler.extract.base.RpcMethod; +import org.apache.dolphinscheduler.extract.base.RpcService; +import org.apache.dolphinscheduler.extract.worker.transportor.GetAppIdRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.GetAppIdResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryResponse; + +@RpcService +public interface IWorkerLogService { + + @RpcMethod + TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest); + + @RpcMethod + TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest); + + @RpcMethod + GetAppIdResponse getAppId(GetAppIdRequest getAppIdRequest); + + @RpcMethod + void removeTaskInstanceLog(String taskInstanceLogAbsolutePath); +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdRequest.java similarity index 73% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdRequest.java index c8a2c570c2..1148a6f24c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdRequest.java @@ -15,16 +15,19 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.protocol; +package org.apache.dolphinscheduler.extract.worker.transportor; -public class RpcProtocolConstants { +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; - public RpcProtocolConstants() { - throw new IllegalStateException("Utility class"); - } +@Data +@NoArgsConstructor +@AllArgsConstructor +public class GetAppIdRequest { - public static final int HEADER_LENGTH = 17; + private int taskInstanceId; - public static final short MAGIC = (short) 0xbabe; + private String logPath; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdResponse.java new file mode 100644 index 0000000000..3b95a4d66e --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/GetAppIdResponse.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class GetAppIdResponse { + + private List appIds; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchRequest.java new file mode 100644 index 0000000000..ca10645152 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchRequest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceDispatchRequest { + + private static final long serialVersionUID = -1L; + + private TaskExecutionContext taskExecutionContext; +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java new file mode 100644 index 0000000000..8845ed9081 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceDispatchResponse { + + private Integer taskInstanceId; + + private boolean dispatchSuccess; + + private String message; + + public static TaskInstanceDispatchResponse success(Integer taskInstanceId) { + return new TaskInstanceDispatchResponse(taskInstanceId, true, "dispatch success"); + } + + public static TaskInstanceDispatchResponse failed(Integer taskInstanceId, String message) { + return new TaskInstanceDispatchResponse(taskInstanceId, false, message); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeAckMessage.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeAckMessage.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java index c6e2a47479..90ae49ab0e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeAckMessage.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java @@ -15,32 +15,26 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.worker.transportor; import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -/** - * task execute running ack command - * from master to worker - */ @Data -@Builder -@AllArgsConstructor @NoArgsConstructor -public class TaskUpdateRuntimeAckMessage implements RequestMessageBuilder { +@AllArgsConstructor +public class TaskInstanceExecutionFinishEventAck { - private boolean success; private int taskInstanceId; + private boolean success; + + public static TaskInstanceExecutionFinishEventAck success(int taskInstanceId) { + return new TaskInstanceExecutionFinishEventAck(taskInstanceId, true); + } - @Override - public MessageType getCommandType() { - return MessageType.TASK_UPDATE_RUNTIME_MESSAGE_ACK; + public static TaskInstanceExecutionFinishEventAck failed(int taskInstanceId) { + return new TaskInstanceExecutionFinishEventAck(taskInstanceId, false); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionInfoEventAck.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionInfoEventAck.java index 5ea8a4f70b..31b080da40 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionInfoEventAck.java @@ -15,30 +15,26 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.worker.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * task savepoint request command - */ @Data @NoArgsConstructor @AllArgsConstructor -public class TaskSavePointRequest implements RequestMessageBuilder { +public class TaskInstanceExecutionInfoEventAck { - /** - * task id - */ + private boolean success; private int taskInstanceId; - @Override - public MessageType getCommandType() { - return MessageType.TASK_SAVEPOINT_REQUEST; + public static TaskInstanceExecutionInfoEventAck success(int taskInstanceId) { + return new TaskInstanceExecutionInfoEventAck(true, taskInstanceId); } + + public static TaskInstanceExecutionInfoEventAck fail(int taskInstanceId) { + return new TaskInstanceExecutionInfoEventAck(false, taskInstanceId); + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointResponse.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java index 32b8293665..aaeb4eb690 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskSavePointResponse.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java @@ -15,30 +15,26 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.worker.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * task savepoint response command - */ @Data @NoArgsConstructor @AllArgsConstructor -public class TaskSavePointResponse implements RequestMessageBuilder { +public class TaskInstanceExecutionRunningEventAck { - /** - * taskInstanceId - */ private int taskInstanceId; + private boolean success; - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; + public static TaskInstanceExecutionRunningEventAck success(int taskInstanceId) { + return new TaskInstanceExecutionRunningEventAck(taskInstanceId, true); } + + public static TaskInstanceExecutionRunningEventAck failed(int taskInstanceId) { + return new TaskInstanceExecutionRunningEventAck(taskInstanceId, false); + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java similarity index 74% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java index f16d951e4e..f4f8dddf12 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java @@ -15,14 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.rpc.serializer; +package org.apache.dolphinscheduler.extract.worker.transportor; -import java.io.IOException; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; -public interface Serializer { +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceKillRequest { - byte[] serialize(T obj) throws IOException; - - T deserialize(byte[] data, Class clz) throws IOException; + private Integer taskInstanceId; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java new file mode 100644 index 0000000000..7ef3050ab7 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + +import java.util.Arrays; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceKillResponse { + + private int taskInstanceId; + + private String host; + + private TaskExecutionStatus status; + + private int processId; + + /** + * other resource manager appId , for example : YARN etc + */ + private List appIds; + + private boolean success; + + private String message; + + public static TaskInstanceKillResponse success(TaskExecutionContext taskExecutionContext) { + TaskInstanceKillResponse taskInstanceKillResponse = new TaskInstanceKillResponse(); + taskInstanceKillResponse.setStatus(taskExecutionContext.getCurrentExecutionStatus()); + if (taskExecutionContext.getAppIds() != null) { + taskInstanceKillResponse + .setAppIds(Arrays.asList(taskExecutionContext.getAppIds().split(TaskConstants.COMMA))); + } + taskInstanceKillResponse.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); + taskInstanceKillResponse.setHost(taskExecutionContext.getHost()); + taskInstanceKillResponse.setProcessId(taskExecutionContext.getProcessId()); + return taskInstanceKillResponse; + } + + public static TaskInstanceKillResponse fail(String message) { + TaskInstanceKillResponse taskInstanceKillResponse = new TaskInstanceKillResponse(); + taskInstanceKillResponse.setSuccess(false); + taskInstanceKillResponse.setMessage(message); + return taskInstanceKillResponse; + } + +} diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadRequest.java similarity index 71% rename from dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadRequest.java index 3a77aa8a0d..1fd3cf2167 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadRequest.java @@ -15,17 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.microbench.common; +package org.apache.dolphinscheduler.extract.worker.transportor; -import org.apache.dolphinscheduler.rpc.base.Rpc; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; -/** - * IUserService - */ -public interface IUserService { +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceLogFileDownloadRequest { - @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) - Boolean say(String s); + private long taskInstanceId; - Integer hi(int num); + private String taskInstanceLogAbsolutePath; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadResponse.java new file mode 100644 index 0000000000..ee805a35b5 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogFileDownloadResponse.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceLogFileDownloadResponse { + + private byte[] logBytes; +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryRequest.java similarity index 71% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryRequest.java index df0df0eb6f..321353e5d3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryRequest.java @@ -15,27 +15,21 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.worker.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@AllArgsConstructor @NoArgsConstructor -public class GetAppIdRequest implements RequestMessageBuilder { +@AllArgsConstructor +public class TaskInstanceLogPageQueryRequest { private int taskInstanceId; - private String logPath; - - @Override - public MessageType getCommandType() { - return MessageType.GET_APP_ID_REQUEST; - } + private String taskInstanceLogAbsolutePath; + private int skipLineNum; + private int limit; } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryResponse.java new file mode 100644 index 0000000000..1e0d481ddd --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceLogPageQueryResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceLogPageQueryResponse { + + private String logContent; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseRequest.java new file mode 100644 index 0000000000..8026f1d85d --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstancePauseRequest { + + private Integer taskInstanceId; +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseResponse.java new file mode 100644 index 0000000000..cf26364829 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstancePauseResponse.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstancePauseResponse { + + private boolean success; + + private String message; + + public static TaskInstancePauseResponse success() { + return new TaskInstancePauseResponse(true, null); + } + + public static TaskInstancePauseResponse fail(String message) { + return new TaskInstancePauseResponse(false, message); + } + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java new file mode 100644 index 0000000000..3d649fa8d8 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskInstanceTriggerSavepointRequest { + + private int taskInstanceId; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java similarity index 67% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequest.java rename to dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java index 23916f140a..82093e1cf9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java @@ -15,30 +15,27 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; +package org.apache.dolphinscheduler.extract.worker.transportor; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * get log bytes request command - */ @Data @NoArgsConstructor @AllArgsConstructor -public class GetLogBytesRequest implements RequestMessageBuilder { +public class TaskInstanceTriggerSavepointResponse { + + public boolean success; - /** - * log path - */ - private String path; + public String message; - @Override - public MessageType getCommandType() { - return MessageType.GET_LOG_BYTES_REQUEST; + public static TaskInstanceTriggerSavepointResponse success() { + return new TaskInstanceTriggerSavepointResponse(true, null); } + + public static TaskInstanceTriggerSavepointResponse fail(String message) { + return new TaskInstanceTriggerSavepointResponse(false, message); + } + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostRequest.java new file mode 100644 index 0000000000..a8a02c7d14 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostRequest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class UpdateWorkflowHostRequest { + + private int taskInstanceId; + + private String workflowHost; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostResponse.java new file mode 100644 index 0000000000..f73ac59cc9 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/UpdateWorkflowHostResponse.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.worker.transportor; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class UpdateWorkflowHostResponse { + + private boolean success; + private String message; + + public static UpdateWorkflowHostResponse success() { + return new UpdateWorkflowHostResponse(true, null); + } + + public static UpdateWorkflowHostResponse failed(String message) { + return new UpdateWorkflowHostResponse(false, message); + } + +} diff --git a/dolphinscheduler-extract/pom.xml b/dolphinscheduler-extract/pom.xml new file mode 100644 index 0000000000..dca4fd6623 --- /dev/null +++ b/dolphinscheduler-extract/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + dev-SNAPSHOT + + + dolphinscheduler-extract + pom + + dolphinscheduler-extract-base + dolphinscheduler-extract-master + dolphinscheduler-extract-worker + dolphinscheduler-extract-alert + + + diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml index 8795753dda..31627d4b7f 100644 --- a/dolphinscheduler-master/pom.xml +++ b/dolphinscheduler-master/pom.xml @@ -54,6 +54,7 @@ org.apache.dolphinscheduler dolphinscheduler-service + org.apache.dolphinscheduler dolphinscheduler-registry-all @@ -73,6 +74,16 @@ dolphinscheduler-storage-all + + org.apache.dolphinscheduler + dolphinscheduler-extract-master + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-worker + + org.springframework.boot spring-boot-starter-cache diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 1bbe919353..7169203834 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,8 +24,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.scheduler.api.SchedulerApi; import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient; -import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; +import org.apache.dolphinscheduler.server.master.rpc.MasterRpcServer; import org.apache.dolphinscheduler.server.master.runner.EventExecuteService; import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap; @@ -72,10 +71,7 @@ public class MasterServer implements IStoppable { private FailoverExecuteThread failoverExecuteThread; @Autowired - private MasterRPCServer masterRPCServer; - - @Autowired - private MasterRpcClient masterRpcClient; + private MasterRpcServer masterRPCServer; public static void main(String[] args) { Thread.currentThread().setName(Constants.THREAD_NAME_MASTER_SERVER); @@ -89,7 +85,6 @@ public class MasterServer implements IStoppable { public void run() throws SchedulerException { // init rpc server this.masterRPCServer.start(); - this.masterRpcClient.start(); // install task plugin this.taskPluginManager.loadPlugin(); @@ -129,8 +124,7 @@ public class MasterServer implements IStoppable { try ( SchedulerApi closedSchedulerApi = schedulerApi; MasterSchedulerBootstrap closedSchedulerBootstrap = masterSchedulerBootstrap; - MasterRPCServer closedRpcServer = masterRPCServer; - MasterRpcClient closedRpcClient = masterRpcClient; + MasterRpcServer closedRpcServer = masterRPCServer; MasterRegistryClient closedMasterRegistryClient = masterRegistryClient; // close spring Context and will invoke method with @PreDestroy annotation to destroy beans. // like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index 366de70323..f1b8485c69 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -20,8 +20,6 @@ package org.apache.dolphinscheduler.server.master.config; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelector; import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; @@ -101,10 +99,6 @@ public class MasterConfig implements Validator { private Duration workerGroupRefreshInterval = Duration.ofSeconds(10L); - private NettyClientConfig masterRpcClientConfig = new NettyClientConfig(); - - private NettyServerConfig masterRpcServerConfig = new NettyServerConfig(); - // ip:listenPort private String masterAddress; @@ -187,7 +181,5 @@ public class MasterConfig implements Validator { log.info("Master config: masterAddress -> {} ", masterAddress); log.info("Master config: masterRegistryPath -> {} ", masterRegistryPath); log.info("Master config: workerGroupRefreshInterval -> {} ", workerGroupRefreshInterval); - log.info("Master config: masterRpcServerConfig -> {} ", masterRpcServerConfig); - log.info("Master config: masterRpcClientConfig -> {} ", masterRpcClientConfig); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java index 28f26d5cb8..85d8cf49e0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.master.controller; -import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; import org.apache.dolphinscheduler.server.master.service.ExecutingService; import java.util.Optional; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java index e0b49a531a..8ad4013868 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java @@ -20,8 +20,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.context; import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import lombok.AllArgsConstructor; @@ -35,16 +34,8 @@ import lombok.NoArgsConstructor; @AllArgsConstructor public class ExecutionContext { - /** - * host - */ private Host host; - /** - * command - */ - private Message message; - private TaskInstance taskInstance; private ExecutorType executorType; @@ -54,12 +45,11 @@ public class ExecutionContext { */ private String workerGroup; - public ExecutionContext(Message message, ExecutorType executorType, TaskInstance taskInstance) { - this(message, executorType, DEFAULT_WORKER_GROUP, taskInstance); + public ExecutionContext(ExecutorType executorType, TaskInstance taskInstance) { + this(executorType, DEFAULT_WORKER_GROUP, taskInstance); } - public ExecutionContext(Message message, ExecutorType executorType, String workerGroup, TaskInstance taskInstance) { - this.message = message; + public ExecutionContext(ExecutorType executorType, String workerGroup, TaskInstance taskInstance) { this.executorType = executorType; this.workerGroup = workerGroup; this.taskInstance = taskInstance; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java deleted file mode 100644 index 41b1837e5d..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/AbstractExecutorManager.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.dispatch.executor; - -import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; - -/** - * abstract executor manager - */ -public abstract class AbstractExecutorManager implements ExecutorManager { - - /** - * before execute , add time monitor ， timeout - * - * @param context context - * @throws ExecuteException if error throws ExecuteException - */ - @Override - public void beforeExecute(ExecutionContext context) throws ExecuteException { - } - - /** - * after execute , add dispatch monitor - * @param context context - * @throws ExecuteException if error throws ExecuteException - */ - @Override - public void afterExecute(ExecutionContext context) throws ExecuteException { - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java deleted file mode 100644 index 9f1e02f151..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/ExecutorManager.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.dispatch.executor; - -import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; - -/** - * executor manager - */ -public interface ExecutorManager { - - /** - * before execute - * - * @param executeContext executeContext - * @throws ExecuteException if error throws ExecuteException - */ - void beforeExecute(ExecutionContext executeContext) throws ExecuteException; - - /** - * execute task - * - * @param context context - * @return T - * @throws ExecuteException if error throws ExecuteException - */ - void execute(ExecutionContext context) throws ExecuteException; - - /** - * execute task directly without retry - * @param context context - * @throws ExecuteException if error throws ExecuteException - */ - void executeDirectly(ExecutionContext context) throws ExecuteException; - - /** - * after execute - * @param context context - * @throws ExecuteException if error throws ExecuteException - */ - void afterExecute(ExecutionContext context) throws ExecuteException; -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java deleted file mode 100644 index 7ceb6e2977..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.dispatch.executor; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; -import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; -import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; -import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import javax.annotation.PostConstruct; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * netty executor manager - */ -@Service -@Slf4j -public class NettyExecutorManager extends AbstractExecutorManager { - - /** - * server node manager - */ - @Autowired - private ServerNodeManager serverNodeManager; - - @Autowired - private List nettyRequestProcessors; - - /** - * netty remote client - */ - private final NettyRemotingClient nettyRemotingClient; - - /** - * constructor - */ - public NettyExecutorManager() { - final NettyClientConfig clientConfig = new NettyClientConfig(); - this.nettyRemotingClient = new NettyRemotingClient(clientConfig); - } - - @PostConstruct - public void init() { - for (NettyRequestProcessor nettyRequestProcessor : nettyRequestProcessors) { - this.nettyRemotingClient.registerProcessor(nettyRequestProcessor); - } - } - - /** - * execute logic - * - * @param context context - * @return result - * @throws ExecuteException if error throws ExecuteException - */ - @Override - public void execute(ExecutionContext context) throws ExecuteException { - // all nodes - Set allNodes = getAllNodes(context); - // fail nodes - Set failNodeSet = new HashSet<>(); - // build command accord executeContext - Message message = context.getMessage(); - // execute task host - Host host = context.getHost(); - for (int i = 0; i < allNodes.size(); i++) { - try { - doExecute(host, message); - context.setHost(host); - // We set the host to taskInstance to avoid when the worker down, this taskInstance may not be - // failovered, due to the taskInstance's host - // is not belongs to the down worker ISSUE-10842. - context.getTaskInstance().setHost(host.getAddress()); - return; - } catch (ExecuteException ex) { - log.error("Execute command {} error", message, ex); - try { - failNodeSet.add(host.getAddress()); - Set tmpAllIps = new HashSet<>(allNodes); - Collection remained = CollectionUtils.subtract(tmpAllIps, failNodeSet); - if (CollectionUtils.isNotEmpty(remained)) { - host = Host.of(remained.iterator().next()); - log.error("retry execute command : {} host : {}", message, host); - } else { - throw new ExecuteException("fail after try all nodes"); - } - } catch (Throwable t) { - throw new ExecuteException("fail after try all nodes"); - } - } - } - } - - @Override - public void executeDirectly(ExecutionContext context) throws ExecuteException { - Host host = context.getHost(); - doExecute(host, context.getMessage()); - } - - /** - * execute logic - * - * @param host host - * @param message command - * @throws ExecuteException if error throws ExecuteException - */ - public void doExecute(final Host host, final Message message) throws ExecuteException { - // retry count，default retry 3 - int retryCount = 3; - boolean success = false; - do { - try { - nettyRemotingClient.send(host, message); - success = true; - } catch (Exception ex) { - log.error("Send command to {} error, command: {}", host, message, ex); - retryCount--; - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); - } - } while (retryCount >= 0 && !success); - - if (!success) { - throw new ExecuteException(String.format("send command : %s to %s error", message, host)); - } - } - - /** - * get all nodes - * - * @param context context - * @return nodes - */ - private Set getAllNodes(ExecutionContext context) throws WorkerGroupNotFoundException { - Set nodes = Collections.emptySet(); - /** - * executor type - */ - ExecutorType executorType = context.getExecutorType(); - switch (executorType) { - case WORKER: - nodes = serverNodeManager.getWorkerGroupNodes(context.getWorkerGroup()); - break; - case CLIENT: - break; - default: - throw new IllegalArgumentException("invalid executor type : " + executorType); - - } - return nodes; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java index 91fadf31bd..5ca63f12e3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java index 9ce087197f..663b5c82b2 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/HostManager.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import java.util.Optional; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java index 0daaebd10c..a6cb9352a8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWeight; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java index b3cb2ae75c..2cd65be1f1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Constants; +import org.apache.dolphinscheduler.extract.base.utils.Host; /** * host weight diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java index 6515464b89..f25b621840 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Host; /** * host worker diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java index 4afaadf460..25ed3283ed 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java @@ -22,7 +22,9 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessageAck; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; @@ -111,13 +113,11 @@ public class TaskDelayEventHandler implements TaskEventHandler { private void sendAckToWorker(TaskEvent taskEvent) { // If event handle success, send ack to worker to otherwise the worker will retry this event - TaskExecuteRunningMessageAck taskExecuteRunningMessageAck = - new TaskExecuteRunningMessageAck(true, - taskEvent.getTaskInstanceId(), - masterConfig.getMasterAddress(), - taskEvent.getWorkerAddress(), - System.currentTimeMillis()); - taskEvent.getChannel().writeAndFlush(taskExecuteRunningMessageAck.convert2Command()); + ITaskInstanceExecutionEventAckListener instanceExecutionEventAckListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskEvent.getWorkerAddress(), ITaskInstanceExecutionEventAckListener.class); + instanceExecutionEventAckListener.handleTaskInstanceExecutionRunningEventAck( + TaskInstanceExecutionRunningEventAck.success(taskEvent.getTaskInstanceId())); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java index be38e40ece..f331dfd360 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java @@ -22,7 +22,9 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessageAck; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionFinishEventAck; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; @@ -36,8 +38,6 @@ import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - @Component public class TaskResultEventHandler implements TaskEventHandler { @@ -115,17 +115,11 @@ public class TaskResultEventHandler implements TaskEventHandler { } public void sendAckToWorker(TaskEvent taskEvent) { - Channel channel = taskEvent.getChannel(); - if (channel == null) { - return; - } - // we didn't set the receiver address, since the ack doen's need to retry - TaskExecuteResultMessageAck taskExecuteAckMessage = new TaskExecuteResultMessageAck(true, - taskEvent.getTaskInstanceId(), - masterConfig.getMasterAddress(), - taskEvent.getWorkerAddress(), - System.currentTimeMillis()); - channel.writeAndFlush(taskExecuteAckMessage.convert2Command()); + ITaskInstanceExecutionEventAckListener instanceExecutionEventAckListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskEvent.getWorkerAddress(), ITaskInstanceExecutionEventAckListener.class); + instanceExecutionEventAckListener.handleTaskInstanceExecutionFinishEventAck( + TaskInstanceExecutionFinishEventAck.success(taskEvent.getTaskInstanceId())); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java index eb82261b23..03300f2ddb 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java @@ -22,7 +22,9 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessageAck; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; @@ -107,14 +109,11 @@ public class TaskRunningEventHandler implements TaskEventHandler { private void sendAckToWorker(TaskEvent taskEvent) { // If event handle success, send ack to worker to otherwise the worker will retry this event - TaskExecuteRunningMessageAck taskExecuteRunningMessageAck = - new TaskExecuteRunningMessageAck( - true, - taskEvent.getTaskInstanceId(), - masterConfig.getMasterAddress(), - taskEvent.getWorkerAddress(), - System.currentTimeMillis()); - taskEvent.getChannel().writeAndFlush(taskExecuteRunningMessageAck.convert2Command()); + ITaskInstanceExecutionEventAckListener instanceExecutionEventAckListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskEvent.getWorkerAddress(), ITaskInstanceExecutionEventAckListener.class); + instanceExecutionEventAckListener.handleTaskInstanceExecutionRunningEventAck( + TaskInstanceExecutionRunningEventAck.success(taskEvent.getTaskInstanceId())); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java index cf9fc262ad..731a16b619 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.master.event; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; @@ -76,7 +75,7 @@ public class TaskStateEventHandler implements StateEventHandler { log.info("The task instance need to release task Group: {}", task.getTaskGroupId()); try { workflowExecuteRunnable.releaseTaskGroup(task); - } catch (RemotingException | InterruptedException e) { + } catch (Exception e) { throw new StateEventHandleException("Release task group failed", e); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskUpdatePidEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskUpdatePidEventHandler.java index 0c4800b8fd..f684322875 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskUpdatePidEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskUpdatePidEventHandler.java @@ -21,7 +21,9 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils; -import org.apache.dolphinscheduler.remote.command.task.TaskUpdateRuntimeAckMessage; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionInfoEventAck; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; @@ -50,7 +52,7 @@ public class TaskUpdatePidEventHandler implements TaskEventHandler { int processInstanceId = taskEvent.getProcessInstanceId(); WorkflowExecuteRunnable workflowExecuteRunnable = - this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); + processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); if (workflowExecuteRunnable == null) { sendAckToWorker(taskEvent); throw new TaskEventHandleError( @@ -91,9 +93,11 @@ public class TaskUpdatePidEventHandler implements TaskEventHandler { private void sendAckToWorker(TaskEvent taskEvent) { // If event handle success, send ack to worker to otherwise the worker will retry this event - TaskUpdateRuntimeAckMessage taskUpdateRuntimeAckMessage = - new TaskUpdateRuntimeAckMessage(true, taskEvent.getTaskInstanceId()); - taskEvent.getChannel().writeAndFlush(taskUpdateRuntimeAckMessage.convert2Command()); + ITaskInstanceExecutionEventAckListener instanceExecutionEventAckListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskEvent.getWorkerAddress(), ITaskInstanceExecutionEventAckListener.class); + instanceExecutionEventAckListener.handleTaskInstanceExecutionInfoEventAck( + TaskInstanceExecutionInfoEventAck.success(taskEvent.getTaskInstanceId())); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java index dc1341db1d..2db3d04c0b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java @@ -90,12 +90,12 @@ public class ProcessInstanceMetrics { .increment(); } - public void cleanUpProcessInstanceCountMetricsByDefinitionCode(final String processDefinitionCode) { + public void cleanUpProcessInstanceCountMetricsByDefinitionCode(final Long processDefinitionCode) { for (final String state : processInstanceStates) { final Counter counter = Metrics.globalRegistry.counter( "ds.workflow.instance.count", "state", state, - "process.definition.code", processDefinitionCode); + "process.definition.code", String.valueOf(processDefinitionCode)); Metrics.globalRegistry.remove(counter); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskDispatchProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskDispatchProcessor.java deleted file mode 100644 index e70199152a..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskDispatchProcessor.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchRequest; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchResponse; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.runner.MasterDelayTaskExecuteRunnableDelayQueue; -import org.apache.dolphinscheduler.server.master.runner.execute.MasterDelayTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecuteRunnableFactoryBuilder; -import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutionContextHolder; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Slf4j -@Component -public class MasterTaskDispatchProcessor implements MasterRpcProcessor { - - @Autowired - private MasterTaskExecuteRunnableFactoryBuilder masterTaskExecuteRunnableFactoryBuilder; - - @Autowired - private MasterMessageSenderManager masterMessageSenderManager; - - @Autowired - private MasterDelayTaskExecuteRunnableDelayQueue masterDelayTaskExecuteRunnableDelayQueue; - - @Override - public void process(Channel channel, Message message) { - TaskDispatchRequest taskDispatchRequest = JSONUtils.parseObject(message.getBody(), TaskDispatchRequest.class); - log.info("Receive task dispatch request, command: {}", taskDispatchRequest); - TaskExecutionContext taskExecutionContext = taskDispatchRequest.getTaskExecutionContext(); - taskExecutionContext.setLogPath(LogUtils.getTaskInstanceLogFullPath(taskExecutionContext)); - try { - // Since we need to make sure remove MDC key after cache, so we use finally to remove MDC key - LogUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), - taskExecutionContext.getTaskInstanceId()); - LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath()); - MasterTaskExecutionContextHolder.putTaskExecutionContext(taskExecutionContext); - // todo: calculate the delay in master dispatcher then we don't need to use a queue to store the task - long remainTime = - DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - TimeUnit.SECONDS.toMillis(taskExecutionContext.getDelayTime())); - - // todo: move this to the master delay queue - if (remainTime > 0) { - log.info("Current taskInstance: {} is choose delay execution, delay time: {}ms, remainTime: {}ms", - taskExecutionContext.getTaskName(), - TimeUnit.SECONDS.toMillis(taskExecutionContext.getDelayTime()), remainTime); - taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.DELAY_EXECUTION); - masterMessageSenderManager.getMasterTaskExecuteResultMessageSender() - .sendMessage(taskExecutionContext); - } - - MasterDelayTaskExecuteRunnable masterDelayTaskExecuteRunnable = masterTaskExecuteRunnableFactoryBuilder - .createWorkerDelayTaskExecuteRunnableFactory(taskExecutionContext.getTaskType()) - .createWorkerTaskExecuteRunnable(taskExecutionContext); - if (masterDelayTaskExecuteRunnableDelayQueue - .submitMasterDelayTaskExecuteRunnable(masterDelayTaskExecuteRunnable)) { - log.info( - "Submit task: {} to MasterDelayTaskExecuteRunnableDelayQueue success", - taskExecutionContext.getTaskName()); - sendDispatchSuccessResult(channel, message, taskExecutionContext); - } else { - log.error( - "Submit task: {} to MasterDelayTaskExecuteRunnableDelayQueue failed, current task waiting queue size: {} is full", - taskExecutionContext.getTaskName(), masterDelayTaskExecuteRunnableDelayQueue.size()); - sendDispatchRejectResult(channel, message, taskExecutionContext); - } - } catch (Exception ex) { - log.error("Handle task dispatch request error, command: {}", taskDispatchRequest, ex); - MasterTaskExecutionContextHolder.removeTaskExecutionContext(taskExecutionContext.getTaskInstanceId()); - sendDispatchFailedResult(channel, message, taskExecutionContext, ex); - } finally { - LogUtils.removeWorkflowAndTaskInstanceIdMDC(); - LogUtils.removeTaskInstanceLogFullPathMDC(); - } - } - - private void sendDispatchSuccessResult(Channel channel, Message dispatchRequest, - TaskExecutionContext taskExecutionContext) { - TaskDispatchResponse taskDispatchResponse = - TaskDispatchResponse.success(taskExecutionContext.getTaskInstanceId()); - channel.writeAndFlush(taskDispatchResponse.convert2Command(dispatchRequest.getOpaque())); - } - - private void sendDispatchRejectResult(Channel channel, Message dispatchRequest, - TaskExecutionContext taskExecutionContext) { - TaskDispatchResponse taskDispatchResponse = - TaskDispatchResponse.failed(taskExecutionContext.getTaskInstanceId(), "Task dispatch queue is full"); - channel.writeAndFlush(taskDispatchResponse.convert2Command(dispatchRequest.getOpaque())); - } - - private void sendDispatchFailedResult(Channel channel, Message dispatchRequest, - TaskExecutionContext taskExecutionContext, Throwable throwable) { - TaskDispatchResponse taskDispatchResponse = - TaskDispatchResponse.failed(taskExecutionContext.getTaskInstanceId(), throwable.getMessage()); - channel.writeAndFlush(taskDispatchResponse.convert2Command(dispatchRequest.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_DISPATCH_REQUEST; - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResultProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResultProcessor.java deleted file mode 100644 index bf48d62b69..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResultProcessor.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessage; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute response processor - */ -@Component -@Slf4j -public class TaskExecuteResultProcessor implements MasterRpcProcessor { - - @Autowired - private TaskEventService taskEventService; - - /** - * task final result response - * need master process , state persistence - * - * @param channel channel - * @param message command - */ - @Override - public void process(Channel channel, Message message) { - TaskExecuteResultMessage taskExecuteResultMessage = JSONUtils.parseObject(message.getBody(), - TaskExecuteResultMessage.class); - TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, - channel, - taskExecuteResultMessage.getMessageSenderAddress()); - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setWorkflowAndTaskInstanceIDMDC( - taskResultEvent.getProcessInstanceId(), taskResultEvent.getTaskInstanceId())) { - log.info("Received task execute result, event: {}", taskResultEvent); - - taskEventService.addEvent(taskResultEvent); - } - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RESULT_MESSAGE; - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java deleted file mode 100644 index e768693bea..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessage; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute running processor - */ -@Component -@Slf4j -public class TaskExecuteRunningProcessor implements MasterRpcProcessor { - - @Autowired - private TaskEventService taskEventService; - - /** - * task ack process - * - * @param channel channel channel - * @param message command TaskExecuteAckCommand - */ - @Override - public void process(Channel channel, Message message) { - TaskExecuteRunningMessage taskExecuteRunningMessage = - JSONUtils.parseObject(message.getBody(), TaskExecuteRunningMessage.class); - log.info("taskExecuteRunningCommand: {}", taskExecuteRunningMessage); - - TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, - channel, - taskExecuteRunningMessage.getMessageSenderAddress()); - taskEventService.addEvent(taskEvent); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RUNNING_MESSAGE; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java deleted file mode 100644 index 67c69555d1..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.TaskDefinition; -import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteStartMessage; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteThreadPool; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute start processor, from api to master - */ -@Component -@Slf4j -public class TaskExecuteStartProcessor implements MasterRpcProcessor { - - @Autowired - private StreamTaskExecuteThreadPool streamTaskExecuteThreadPool; - - @Autowired - private TaskDefinitionDao taskDefinitionDao; - - @Override - public void process(Channel channel, Message message) { - TaskExecuteStartMessage taskExecuteStartMessage = - JSONUtils.parseObject(message.getBody(), TaskExecuteStartMessage.class); - log.info("taskExecuteStartCommand: {}", taskExecuteStartMessage); - - TaskDefinition taskDefinition = taskDefinitionDao.findTaskDefinition( - taskExecuteStartMessage.getTaskDefinitionCode(), taskExecuteStartMessage.getTaskDefinitionVersion()); - if (taskDefinition == null) { - log.error("Task definition can not be found, taskDefinitionCode:{}, taskDefinitionVersion:{}", - taskExecuteStartMessage.getTaskDefinitionCode(), - taskExecuteStartMessage.getTaskDefinitionVersion()); - return; - } - streamTaskExecuteThreadPool.execute(new StreamTaskExecuteRunnable(taskDefinition, taskExecuteStartMessage)); - - // response - Message response = new Message(message.getOpaque()); - response.setType(MessageType.TASK_EXECUTE_START); - response.setBody(new byte[0]); - channel.writeAndFlush(response); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_START; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java deleted file mode 100644 index 76ec751103..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskKillResponse; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task response processor - */ -@Component -@Slf4j -public class TaskKillResponseProcessor implements MasterRpcProcessor { - - /** - * task final result response - * need master process , state persistence - * - * @param channel channel - * @param message command - */ - @Override - public void process(Channel channel, Message message) { - TaskKillResponse responseCommand = JSONUtils.parseObject(message.getBody(), TaskKillResponse.class); - log.info("[TaskInstance-{}] Received task kill response command : {}", - responseCommand.getTaskInstanceId(), responseCommand); - } - - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskUpdateRuntimeProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskUpdateRuntimeProcessor.java deleted file mode 100644 index 5edf3acf6e..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskUpdateRuntimeProcessor.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskUpdateRuntimeMessage; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute running processor - */ -@Component -@Slf4j -public class TaskUpdateRuntimeProcessor implements MasterRpcProcessor { - - @Autowired - private TaskEventService taskEventService; - - /** - * task ack process - * - * @param channel channel channel - * @param message command TaskExecuteAckCommand - */ - @Override - public void process(Channel channel, Message message) { - TaskUpdateRuntimeMessage taskUpdatePidRequest = - JSONUtils.parseObject(message.getBody(), TaskUpdateRuntimeMessage.class); - log.info("taskUpdatePidCommand: {}", taskUpdatePidRequest); - - TaskEvent taskEvent = TaskEvent.newUpdatePidEvent(taskUpdatePidRequest, - channel, - taskUpdatePidRequest.getMessageSenderAddress()); - taskEventService.addEvent(taskEvent); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_UPDATE_RUNTIME_MESSAGE; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java deleted file mode 100644 index 036cc315ff..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowExecutingDataRequest; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowExecutingDataResponse; -import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.service.ExecutingService; - -import java.util.Optional; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * workflow executing data process from api/master - */ -@Component -@Slf4j -public class WorkflowExecutingDataRequestProcessor implements MasterRpcProcessor { - - @Autowired - private ExecutingService executingService; - - @Override - public void process(Channel channel, Message message) { - WorkflowExecutingDataRequest requestCommand = - JSONUtils.parseObject(message.getBody(), WorkflowExecutingDataRequest.class); - - log.info("received command, processInstanceId:{}", requestCommand.getProcessInstanceId()); - - Optional workflowExecuteDtoOptional = - executingService.queryWorkflowExecutingData(requestCommand.getProcessInstanceId()); - - WorkflowExecutingDataResponse responseCommand = new WorkflowExecutingDataResponse(); - workflowExecuteDtoOptional.ifPresent(responseCommand::setWorkflowExecuteDto); - channel.writeAndFlush(responseCommand.convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.WORKFLOW_EXECUTING_DATA_REQUEST; - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowMetricsCleanUpProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowMetricsCleanUpProcessor.java deleted file mode 100644 index dae969c4ad..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowMetricsCleanUpProcessor.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowMetricsCleanUpRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Component -public class WorkflowMetricsCleanUpProcessor implements MasterRpcProcessor { - - @Override - public void process(Channel channel, Message message) { - WorkflowMetricsCleanUpRequest workflowMetricsCleanUpRequest = - JSONUtils.parseObject(message.getBody(), WorkflowMetricsCleanUpRequest.class); - - ProcessInstanceMetrics.cleanUpProcessInstanceCountMetricsByDefinitionCode( - workflowMetricsCleanUpRequest.getProcessDefinitionCode()); - } - - @Override - public MessageType getCommandType() { - return MessageType.WORKFLOW_METRICS_CLEANUP; - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java index 11329732c9..03f254a09c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master.processor.queue; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.StateEventResponse; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; @@ -39,8 +38,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - @Component @Slf4j public class StateEventResponseService { @@ -74,11 +71,11 @@ public class StateEventResponseService { List remainEvents = new ArrayList<>(eventQueue.size()); eventQueue.drainTo(remainEvents); for (StateEvent event : remainEvents) { - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), - event.getTaskInstanceId())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId()); this.persist(event); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } } @@ -118,32 +115,24 @@ public class StateEventResponseService { Thread.currentThread().interrupt(); break; } - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), - stateEvent.getTaskInstanceId())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), + stateEvent.getTaskInstanceId()); // if not task , blocking here persist(stateEvent); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } log.info("State event loop service stopped"); } } - private void writeResponse(StateEvent stateEvent) { - Channel channel = stateEvent.getChannel(); - if (channel != null) { - StateEventResponse command = new StateEventResponse(stateEvent.getKey()); - channel.writeAndFlush(command.convert2Command()); - } - } - private void persist(StateEvent stateEvent) { try { if (!this.processInstanceExecCacheManager.contains(stateEvent.getProcessInstanceId())) { log.warn("Persist event into workflow execute thread error, " + "cannot find the workflow instance from cache manager, event: {}", stateEvent); - writeResponse(stateEvent); return; } @@ -160,8 +149,6 @@ public class StateEventResponseService { default: } workflowExecuteThreadPool.submitStateEvent(stateEvent); - // this response is not needed. - writeResponse(stateEvent); } catch (Exception e) { log.error("Persist event queue error, event: {}", stateEvent, e); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java index ff250ed403..e55dfbdd5d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java @@ -19,10 +19,10 @@ package org.apache.dolphinscheduler.server.master.processor.queue; import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionInfoEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionRunningEvent; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessage; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessage; -import org.apache.dolphinscheduler.remote.command.task.TaskUpdateRuntimeMessage; import java.util.Date; @@ -30,7 +30,6 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import io.netty.channel.Channel; @Data @Builder @@ -95,11 +94,6 @@ public class TaskEvent { private int cacheTaskInstanceId; - /** - * channel - */ - private Channel channel; - private int processInstanceId; public static TaskEvent newDispatchEvent(int processInstanceId, int taskInstanceId, String workerAddress) { @@ -111,7 +105,7 @@ public class TaskEvent { return event; } - public static TaskEvent newRunningEvent(TaskExecuteRunningMessage command, Channel channel, String workerAddress) { + public static TaskEvent newRunningEvent(TaskInstanceExecutionRunningEvent command) { TaskEvent event = new TaskEvent(); event.setProcessInstanceId(command.getProcessInstanceId()); event.setTaskInstanceId(command.getTaskInstanceId()); @@ -120,13 +114,12 @@ public class TaskEvent { event.setExecutePath(command.getExecutePath()); event.setLogPath(command.getLogPath()); event.setAppIds(command.getAppIds()); - event.setChannel(channel); - event.setWorkerAddress(workerAddress); + event.setWorkerAddress(command.getHost()); event.setEvent(TaskEventType.RUNNING); return event; } - public static TaskEvent newResultEvent(TaskExecuteResultMessage command, Channel channel, String workerAddress) { + public static TaskEvent newResultEvent(TaskInstanceExecutionFinishEvent command) { TaskEvent event = new TaskEvent(); event.setProcessInstanceId(command.getProcessInstanceId()); event.setTaskInstanceId(command.getTaskInstanceId()); @@ -138,8 +131,7 @@ public class TaskEvent { event.setProcessId(command.getProcessId()); event.setAppIds(command.getAppIds()); event.setVarPool(command.getVarPool()); - event.setChannel(channel); - event.setWorkerAddress(workerAddress); + event.setWorkerAddress(command.getHost()); event.setEvent(TaskEventType.RESULT); return event; } @@ -153,14 +145,13 @@ public class TaskEvent { return event; } - public static TaskEvent newUpdatePidEvent(TaskUpdateRuntimeMessage command, Channel channel, String workerAddress) { + public static TaskEvent newUpdatePidEvent(TaskInstanceExecutionInfoEvent command) { TaskEvent event = new TaskEvent(); event.setProcessInstanceId(command.getProcessInstanceId()); event.setTaskInstanceId(command.getTaskInstanceId()); event.setStartTime(DateUtils.timeStampToDate(command.getStartTime())); event.setLogPath(command.getLogPath()); - event.setChannel(channel); - event.setWorkerAddress(workerAddress); + event.setWorkerAddress(command.getHost()); event.setEvent(TaskEventType.UPDATE_PID); return event; } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java index ab022a762b..78199f517d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java @@ -50,9 +50,8 @@ public class TaskExecuteRunnable implements Runnable { while (!this.events.isEmpty()) { // we handle the task event belongs to one task serial, so if the event comes in wrong order, TaskEvent event = this.events.peek(); - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils - .setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId()); log.info("Handle task event begin: {}", event); taskEventHandlerMap.get(event.getEvent()).handleTaskEvent(event); events.remove(event); @@ -69,6 +68,8 @@ public class TaskExecuteRunnable implements Runnable { log.error("Handle task event error, get a unknown exception, this event will be removed, event: {}", event, unknownException); events.remove(event); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java index 6929317ebd..4089acfd02 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java @@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.registry.api.StrategyType; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; -import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer; +import org.apache.dolphinscheduler.server.master.rpc.MasterRpcServer; import org.apache.dolphinscheduler.server.master.runner.StateWheelExecuteThread; import java.time.Duration; @@ -51,7 +51,7 @@ public class MasterWaitingStrategy implements MasterConnectStrategy { @Autowired private RegistryClient registryClient; @Autowired - private MasterRPCServer masterRPCServer; + private MasterRpcServer masterRPCServer; @Autowired private WorkflowEventQueue workflowEventQueue; @Autowired diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java index 3e2ba337ea..852f73ce73 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java @@ -24,12 +24,12 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; +import org.apache.dolphinscheduler.extract.base.utils.NamedThreadFactory; import org.apache.dolphinscheduler.registry.api.Event; import org.apache.dolphinscheduler.registry.api.Event.Type; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.SubscribeListener; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import org.apache.dolphinscheduler.service.queue.MasterPriorityQueue; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/WorkerRpcProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/ITaskInstanceOperationFunction.java similarity index 85% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/WorkerRpcProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/ITaskInstanceOperationFunction.java index 898d56475c..640ee6be12 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/WorkerRpcProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/ITaskInstanceOperationFunction.java @@ -15,7 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.processor; +package org.apache.dolphinscheduler.server.master.rpc; + +public interface ITaskInstanceOperationFunction { + + Y operate(X x); -public interface WorkerRpcProcessor extends NettyRequestProcessor { } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java new file mode 100644 index 0000000000..22258c613b --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.master.runner.MasterDelayTaskExecuteRunnableDelayQueue; +import org.apache.dolphinscheduler.server.master.runner.execute.MasterDelayTaskExecuteRunnable; +import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecuteRunnableFactoryBuilder; +import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutionContextHolder; + +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class LogicITaskInstanceDispatchOperationFunction + implements + ITaskInstanceOperationFunction { + + @Autowired + private MasterTaskExecuteRunnableFactoryBuilder masterTaskExecuteRunnableFactoryBuilder; + + @Autowired + private MasterDelayTaskExecuteRunnableDelayQueue masterDelayTaskExecuteRunnableDelayQueue; + + @Override + public LogicTaskDispatchResponse operate(LogicTaskDispatchRequest taskDispatchRequest) { + log.info("Received dispatchLogicTask request: {}", taskDispatchRequest); + TaskExecutionContext taskExecutionContext = taskDispatchRequest.getTaskExecutionContext(); + try { + final int taskInstanceId = taskExecutionContext.getTaskInstanceId(); + final int workflowInstanceId = taskExecutionContext.getProcessInstanceId(); + final String taskInstanceName = taskExecutionContext.getTaskName(); + + LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstanceId, taskInstanceId); + LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath()); + + MasterTaskExecutionContextHolder.putTaskExecutionContext(taskExecutionContext); + // todo: calculate the delay in master dispatcher then we don't need to use a queue to store the task + final long remainTime = + DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), + TimeUnit.SECONDS.toMillis(taskExecutionContext.getDelayTime())); + if (remainTime > 0) { + log.info("Current taskInstance: {} is choosing delay execution, delay time: {}/ms, remainTime: {}/ms", + taskExecutionContext.getTaskName(), + TimeUnit.SECONDS.toMillis(taskExecutionContext.getDelayTime()), remainTime); + taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.DELAY_EXECUTION); + // todo: send delay execution message + return LogicTaskDispatchResponse.success(taskExecutionContext.getTaskInstanceId()); + } + + final MasterDelayTaskExecuteRunnable masterDelayTaskExecuteRunnable = + masterTaskExecuteRunnableFactoryBuilder + .createWorkerDelayTaskExecuteRunnableFactory(taskExecutionContext.getTaskType()) + .createWorkerTaskExecuteRunnable(taskExecutionContext); + if (masterDelayTaskExecuteRunnableDelayQueue + .submitMasterDelayTaskExecuteRunnable(masterDelayTaskExecuteRunnable)) { + log.info("Submit LogicTask: {} to MasterDelayTaskExecuteRunnableDelayQueue success", taskInstanceName); + return LogicTaskDispatchResponse.success(taskInstanceId); + } else { + log.error( + "Submit LogicTask: {} to MasterDelayTaskExecuteRunnableDelayQueue failed, current task waiting queue size: {} is full", + taskInstanceName, masterDelayTaskExecuteRunnableDelayQueue.size()); + return LogicTaskDispatchResponse.failed(taskInstanceId, + "MasterDelayTaskExecuteRunnableDelayQueue is full"); + } + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); + LogUtils.removeTaskInstanceLogFullPathMDC(); + } + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskKillProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceKillOperationFunction.java similarity index 62% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskKillProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceKillOperationFunction.java index 37a61e6246..6b43a7690a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskKillProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceKillOperationFunction.java @@ -15,15 +15,11 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.server.master.rpc; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskKillRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.runner.MasterDelayTaskExecuteRunnableDelayQueue; import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecuteRunnable; @@ -35,49 +31,42 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - @Slf4j @Component -public class MasterTaskKillProcessor implements MasterRpcProcessor { +public class LogicITaskInstanceKillOperationFunction + implements + ITaskInstanceOperationFunction { @Autowired private MasterDelayTaskExecuteRunnableDelayQueue masterDelayTaskExecuteRunnableDelayQueue; @Override - public void process(Channel channel, Message message) { - TaskKillRequest taskKillRequest = JSONUtils.parseObject(message.getBody(), TaskKillRequest.class); - log.info("Master receive task kill request: {}", taskKillRequest); - int taskInstanceId = taskKillRequest.getTaskInstanceId(); - try (LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setTaskInstanceIdMDC(taskInstanceId)) { - TaskExecutionContext taskExecutionContext = - MasterTaskExecutionContextHolder.getTaskExecutionContext(taskInstanceId); - if (taskExecutionContext == null) { - log.error("Cannot find the TaskExecutionContext, this task may already been killed"); - return; - } - MasterTaskExecuteRunnable masterTaskExecuteRunnable = + public LogicTaskKillResponse operate(LogicTaskKillRequest taskKillRequest) { + final int taskInstanceId = taskKillRequest.getTaskInstanceId(); + try { + LogUtils.setTaskInstanceIdMDC(taskKillRequest.getTaskInstanceId()); + log.info("Received killLogicTask request: {}", taskKillRequest); + final MasterTaskExecuteRunnable masterTaskExecuteRunnable = MasterTaskExecuteRunnableHolder.getMasterTaskExecuteRunnable(taskInstanceId); if (masterTaskExecuteRunnable == null) { log.error("Cannot find the MasterTaskExecuteRunnable, this task may already been killed"); - return; + return LogicTaskKillResponse.fail("Cannot find the MasterTaskExecuteRunnable"); } try { masterTaskExecuteRunnable.cancelTask(); masterDelayTaskExecuteRunnableDelayQueue .removeMasterDelayTaskExecuteRunnable(masterTaskExecuteRunnable); + return LogicTaskKillResponse.success(); } catch (MasterTaskExecuteException e) { log.error("Cancel MasterTaskExecuteRunnable failed ", e); + return LogicTaskKillResponse.fail("Cancel MasterTaskExecuteRunnable failed: " + e.getMessage()); } finally { + // todo: If cancel failed, we cannot remove the context? MasterTaskExecutionContextHolder.removeTaskExecutionContext(taskInstanceId); MasterTaskExecuteRunnableHolder.removeMasterTaskExecuteRunnable(taskInstanceId); } + } finally { + LogUtils.removeTaskInstanceIdMDC(); } } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_KILL_REQUEST; - } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskPauseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstancePauseOperationFunction.java similarity index 55% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskPauseProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstancePauseOperationFunction.java index 41c9c50d69..912193b91e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/MasterTaskPauseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstancePauseOperationFunction.java @@ -15,15 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.server.master.rpc; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskPauseRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecuteRunnableHolder; @@ -32,33 +29,34 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - @Slf4j @Component -public class MasterTaskPauseProcessor implements MasterRpcProcessor { +public class LogicITaskInstancePauseOperationFunction + implements + ITaskInstanceOperationFunction { @Override - public void process(Channel channel, Message message) { - TaskPauseRequest taskPauseRequest = JSONUtils.parseObject(message.getBody(), TaskPauseRequest.class); - MasterTaskExecuteRunnable masterTaskExecuteRunnable = - MasterTaskExecuteRunnableHolder.getMasterTaskExecuteRunnable(taskPauseRequest.getTaskInstanceId()); - if (masterTaskExecuteRunnable == null) { - log.info("Cannot find the MasterTaskExecuteRunnable"); - return; - } - TaskExecutionContext taskExecutionContext = masterTaskExecuteRunnable.getTaskExecutionContext(); + public LogicTaskPauseResponse operate(LogicTaskPauseRequest taskPauseRequest) { try { + LogUtils.setTaskInstanceIdMDC(taskPauseRequest.getTaskInstanceId()); + final MasterTaskExecuteRunnable masterTaskExecuteRunnable = + MasterTaskExecuteRunnableHolder.getMasterTaskExecuteRunnable(taskPauseRequest.getTaskInstanceId()); + if (masterTaskExecuteRunnable == null) { + log.info("Cannot find the MasterTaskExecuteRunnable"); + return LogicTaskPauseResponse.fail("Cannot find the MasterTaskExecuteRunnable"); + } + final TaskExecutionContext taskExecutionContext = masterTaskExecuteRunnable.getTaskExecutionContext(); LogUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); masterTaskExecuteRunnable.pauseTask(); + return LogicTaskPauseResponse.success(); } catch (MasterTaskExecuteException e) { log.error("Pause MasterTaskExecuteRunnable failed", e); + return LogicTaskPauseResponse.fail("Pause MasterTaskExecuteRunnable failed: " + e.getMessage()); + } finally { + LogUtils.removeTaskInstanceIdMDC(); + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } - @Override - public MessageType getCommandType() { - return MessageType.PAUSE_TASK_INSTANCE; - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java new file mode 100644 index 0000000000..3a07d68364 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class LogicTaskInstanceOperationFunctionManager { + + @Autowired + private LogicITaskInstanceDispatchOperationFunction logicITaskInstanceDispatchOperationFunction; + + @Autowired + private LogicITaskInstanceKillOperationFunction logicITaskInstanceKillOperationFunction; + + @Autowired + private LogicITaskInstancePauseOperationFunction logicITaskInstancePauseOperationFunction; + + @Autowired + private TaskInstanceForceStartOperationFunction taskInstanceForceStartOperationFunction; + + @Autowired + private TaskInstanceWakeupOperationFunction taskInstanceWakeupOperationFunction; + + public LogicITaskInstanceDispatchOperationFunction getLogicTaskInstanceDispatchOperationFunction() { + return logicITaskInstanceDispatchOperationFunction; + } + + public LogicITaskInstanceKillOperationFunction getLogicTaskInstanceKillOperationFunction() { + return logicITaskInstanceKillOperationFunction; + } + + public LogicITaskInstancePauseOperationFunction getLogicTaskInstancePauseOperationFunction() { + return logicITaskInstancePauseOperationFunction; + } + + public TaskInstanceForceStartOperationFunction getTaskInstanceForceStartOperationFunction() { + return taskInstanceForceStartOperationFunction; + } + + public TaskInstanceWakeupOperationFunction getTaskInstanceWakeupOperationFunction() { + return taskInstanceWakeupOperationFunction; + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java new file mode 100644 index 0000000000..c99968d1e9 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class LogicTaskInstanceOperatorImpl implements ILogicTaskInstanceOperator { + + @Autowired + private LogicTaskInstanceOperationFunctionManager logicTaskInstanceOperationFunctionManager; + + @Override + public LogicTaskDispatchResponse dispatchLogicTask(LogicTaskDispatchRequest taskDispatchRequest) { + return logicTaskInstanceOperationFunctionManager.getLogicTaskInstanceDispatchOperationFunction() + .operate(taskDispatchRequest); + } + + @Override + public LogicTaskKillResponse killLogicTask(LogicTaskKillRequest taskKillRequest) { + return logicTaskInstanceOperationFunctionManager.getLogicTaskInstanceKillOperationFunction() + .operate(taskKillRequest); + } + + @Override + public LogicTaskPauseResponse pauseLogicTask(LogicTaskPauseRequest taskPauseRequest) { + return logicTaskInstanceOperationFunctionManager.getLogicTaskInstancePauseOperationFunction() + .operate(taskPauseRequest); + } + + @Override + public TaskInstanceForceStartResponse forceStartTaskInstance(TaskInstanceForceStartRequest taskForceStartRequest) { + return logicTaskInstanceOperationFunctionManager.getTaskInstanceForceStartOperationFunction() + .operate(taskForceStartRequest); + } + + @Override + public void wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest) { + logicTaskInstanceOperationFunctionManager.getTaskInstanceWakeupOperationFunction().operate(taskWakeupRequest); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterCacheServiceImpl.java similarity index 59% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterCacheServiceImpl.java index ee49520a80..08448da811 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterCacheServiceImpl.java @@ -15,50 +15,28 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.server.master.rpc; import org.apache.dolphinscheduler.common.enums.CacheType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.cache.CacheExpireRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; +import org.apache.dolphinscheduler.extract.master.IMasterCacheService; +import org.apache.dolphinscheduler.extract.master.transportor.CacheExpireRequest; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; -import io.netty.channel.Channel; - -/** - * cache process from master/api - */ -@Component @Slf4j -public class CacheProcessor implements MasterRpcProcessor { +@Service +public class MasterCacheServiceImpl implements IMasterCacheService { @Autowired private CacheManager cacheManager; @Override - public void process(Channel channel, Message message) { - CacheExpireRequest cacheExpireRequest = JSONUtils.parseObject(message.getBody(), CacheExpireRequest.class); - - log.info("received command : {}", cacheExpireRequest); - - this.cacheExpire(cacheExpireRequest); - } - - @Override - public MessageType getCommandType() { - return MessageType.CACHE_EXPIRE; - } - - private void cacheExpire(CacheExpireRequest cacheExpireRequest) { - + public void cacheExpire(CacheExpireRequest cacheExpireRequest) { if (cacheExpireRequest.getCacheKey().isEmpty()) { return; } @@ -70,4 +48,5 @@ public class CacheProcessor implements MasterRpcProcessor { log.info("cache evict, type:{}, key:{}", cacheType.getCacheName(), cacheExpireRequest.getCacheKey()); } } + } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java new file mode 100644 index 0000000000..6d908d16ac --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.common.utils.LogUtils; +import org.apache.dolphinscheduler.extract.master.IMasterLogService; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryResponse; + +import java.io.File; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class MasterLogServiceImpl implements IMasterLogService { + + @Override + public LogicTaskInstanceLogFileDownloadResponse getLogicTaskInstanceWholeLogFileBytes(LogicTaskInstanceLogFileDownloadRequest logicTaskInstanceLogFileDownloadRequest) { + byte[] bytes = + LogUtils.getFileContentBytes(logicTaskInstanceLogFileDownloadRequest.getTaskInstanceLogAbsolutePath()); + // todo: if file not exists, return error result + return new LogicTaskInstanceLogFileDownloadResponse(bytes); + } + + @Override + public LogicTaskInstanceLogPageQueryResponse pageQueryLogicTaskInstanceLog(LogicTaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + + List lines = LogUtils.readPartFileContent( + taskInstanceLogPageQueryRequest.getTaskInstanceLogAbsolutePath(), + taskInstanceLogPageQueryRequest.getSkipLineNum(), + taskInstanceLogPageQueryRequest.getLimit()); + + String logContent = LogUtils.rollViewLogLines(lines); + return new LogicTaskInstanceLogPageQueryResponse(logContent); + } + + @Override + public void removeLogicTaskInstanceLog(String taskInstanceLogAbsolutePath) { + File taskLogFile = new File(taskInstanceLogAbsolutePath); + try { + if (taskLogFile.exists()) { + taskLogFile.delete(); + } + } catch (Exception e) { + log.error("Remove LogicTaskInstanceLog error", e); + } + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java deleted file mode 100644 index f5cadb7339..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.rpc; - -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; -import org.apache.dolphinscheduler.server.master.config.MasterConfig; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * Master RPC Server, used to send/receive request to other system. - */ -@Service -@Slf4j -public class MasterRPCServer implements AutoCloseable { - - private NettyRemotingServer nettyRemotingServer; - - @Autowired - private MasterConfig masterConfig; - - @Autowired - private List masterRpcProcessors; - - public void start() { - log.info("Starting Master RPC Server..."); - // init remoting server - NettyServerConfig serverConfig = masterConfig.getMasterRpcServerConfig(); - serverConfig.setListenPort(masterConfig.getListenPort()); - this.nettyRemotingServer = new NettyRemotingServer(serverConfig); - for (MasterRpcProcessor masterRpcProcessor : masterRpcProcessors) { - this.nettyRemotingServer.registerProcessor(masterRpcProcessor); - log.info("Success register netty processor: {}", masterRpcProcessor.getClass().getName()); - } - this.nettyRemotingServer.start(); - log.info("Started Master RPC Server..."); - } - - @Override - public void close() { - log.info("Closing Master RPC Server..."); - this.nettyRemotingServer.close(); - log.info("Closed Master RPC Server..."); - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java deleted file mode 100644 index 15d976cbee..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcClient.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.rpc; - -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.config.MasterConfig; - -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Slf4j -@Component -public class MasterRpcClient implements AutoCloseable { - - @Autowired - private MasterConfig masterConfig; - - private static final long DEFAULT_TIME_OUT_MILLS = 10_000L; - private NettyRemotingClient client; - - public void start() { - client = new NettyRemotingClient(masterConfig.getMasterRpcClientConfig()); - log.info("Success initialized MasterRPCClient..."); - } - - public Message sendSyncCommand(@NonNull Host host, - @NonNull Message rpcMessage) throws RemotingException, InterruptedException { - return client.sendSync(host, rpcMessage, DEFAULT_TIME_OUT_MILLS); - } - - public void send(@NonNull Host host, @NonNull Message message) throws RemotingException { - client.send(host, message); - } - - @Override - public void close() { - if (client != null) { - client.close(); - } - } - -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java new file mode 100644 index 0000000000..91a7448b7c --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRpcServer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class MasterRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable { + + public MasterRpcServer(MasterConfig masterConfig) { + super(NettyRemotingServerFactory.buildNettyRemotingServer(new NettyServerConfig(masterConfig.getListenPort()))); + } + + public void start() { + log.info("Starting MasterRPCServer..."); + nettyRemotingServer.start(); + log.info("Started MasterRPCServer..."); + } + + @Override + public void close() { + log.info("Closing MasterRPCServer..."); + nettyRemotingServer.close(); + log.info("Closed MasterRPCServer..."); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/StreamingTaskOperatorImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/StreamingTaskOperatorImpl.java new file mode 100644 index 0000000000..705b0554c7 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/StreamingTaskOperatorImpl.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; +import org.apache.dolphinscheduler.extract.master.IStreamingTaskOperator; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse; +import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; +import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteThreadPool; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class StreamingTaskOperatorImpl implements IStreamingTaskOperator { + + @Autowired + private StreamTaskExecuteThreadPool streamTaskExecuteThreadPool; + + @Autowired + private TaskDefinitionDao taskDefinitionDao; + + @Override + public StreamingTaskTriggerResponse triggerStreamingTask(StreamingTaskTriggerRequest streamingTaskTriggerRequest) { + log.info("Receive triggerStreamingTask request: {}", streamingTaskTriggerRequest); + + TaskDefinition taskDefinition = + taskDefinitionDao.findTaskDefinition(streamingTaskTriggerRequest.getTaskDefinitionCode(), + streamingTaskTriggerRequest.getTaskDefinitionVersion()); + if (taskDefinition == null) { + log.error("Cannot find the Streaming TaskDefinition: {}", streamingTaskTriggerRequest); + return StreamingTaskTriggerResponse.fail("Cannot find the Streaming TaskDefinition"); + } + streamTaskExecuteThreadPool.execute(new StreamTaskExecuteRunnable(taskDefinition, streamingTaskTriggerRequest)); + return StreamingTaskTriggerResponse.success(); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskForceStartProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java similarity index 54% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskForceStartProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java index 5a1dfb93a6..a15766e4c4 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskForceStartProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java @@ -15,15 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.server.master.rpc; import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskForceStartRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; @@ -32,35 +29,30 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - -@Component @Slf4j -public class TaskForceStartProcessor implements MasterRpcProcessor { +@Component +public class TaskInstanceForceStartOperationFunction + implements + ITaskInstanceOperationFunction { @Autowired private StateEventResponseService stateEventResponseService; @Override - public void process(Channel channel, Message message) { - TaskForceStartRequest taskEventChangeCommand = - JSONUtils.parseObject(message.getBody(), TaskForceStartRequest.class); + public TaskInstanceForceStartResponse operate(TaskInstanceForceStartRequest taskInstanceForceStartRequest) { TaskStateEvent stateEvent = TaskStateEvent.builder() - .processInstanceId(taskEventChangeCommand.getProcessInstanceId()) - .taskInstanceId(taskEventChangeCommand.getTaskInstanceId()) - .key(taskEventChangeCommand.getKey()) + .processInstanceId(taskInstanceForceStartRequest.getProcessInstanceId()) + .taskInstanceId(taskInstanceForceStartRequest.getTaskInstanceId()) + .key(taskInstanceForceStartRequest.getKey()) .type(StateEventType.WAKE_UP_TASK_GROUP) .build(); - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setWorkflowAndTaskInstanceIDMDC( - stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId())) { - log.info("Received task event change command, event: {}", stateEvent); + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId()); + log.info("Received forceStartTaskInstance, event: {}", stateEvent); stateEventResponseService.addEvent2WorkflowExecute(stateEvent); + return TaskInstanceForceStartResponse.success(); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_FORCE_STATE_EVENT_REQUEST; - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceListenerImpl.java new file mode 100644 index 0000000000..e5e67abc49 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceListenerImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionInfoEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionRunningEvent; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; +import org.apache.dolphinscheduler.server.master.runner.listener.TaskInstanceExecutionEventListenerFunctionManager; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class TaskInstanceListenerImpl implements ITaskInstanceExecutionEventListener { + + @Autowired + private TaskInstanceExecutionEventListenerFunctionManager taskInstanceExecutionEventListenerManager; + + @Override + public void onTaskInstanceExecutionRunning(TaskInstanceExecutionRunningEvent taskInstanceExecutionRunningEvent) { + taskInstanceExecutionEventListenerManager + .getTaskInstanceExecutionRunningEventListenFunction() + .handleTaskInstanceExecutionEvent(taskInstanceExecutionRunningEvent); + + } + + @Override + public void onTaskInstanceExecutionFinish(TaskInstanceExecutionFinishEvent taskInstanceExecutionFinishEvent) { + taskInstanceExecutionEventListenerManager.getTaskInstanceExecutionResultEventListenFunction() + .handleTaskInstanceExecutionEvent(taskInstanceExecutionFinishEvent); + } + + @Override + public void onTaskInstanceExecutionInfoUpdate(TaskInstanceExecutionInfoEvent taskInstanceExecutionInfoEvent) { + taskInstanceExecutionEventListenerManager.getTaskInstanceExecutionInfoEventListenFunction() + .handleTaskInstanceExecutionEvent(taskInstanceExecutionInfoEvent); + } + + @Override + public void onWorkflowInstanceInstanceStateChange(WorkflowInstanceStateChangeEvent taskInstanceStateChangeEvent) { + taskInstanceExecutionEventListenerManager.getTaskInstanceStateEventListenFunction() + .handleTaskInstanceExecutionEvent(taskInstanceStateChangeEvent); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskWakeupProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java similarity index 53% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskWakeupProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java index d7cf7b5a8f..9a5b716e90 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskWakeupProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java @@ -15,15 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.server.master.rpc; import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupResponse; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskWakeupRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; @@ -32,39 +29,30 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - -/** - * handle state event received from master/api - */ -@Component @Slf4j -public class TaskWakeupProcessor implements MasterRpcProcessor { +@Component +public class TaskInstanceWakeupOperationFunction + implements + ITaskInstanceOperationFunction { @Autowired private StateEventResponseService stateEventResponseService; @Override - public void process(Channel channel, Message message) { - TaskWakeupRequest taskEventChangeCommand = - JSONUtils.parseObject(message.getBody(), TaskWakeupRequest.class); + public TaskInstanceWakeupResponse operate(TaskInstanceWakeupRequest taskInstanceWakeupRequest) { TaskStateEvent stateEvent = TaskStateEvent.builder() - .processInstanceId(taskEventChangeCommand.getProcessInstanceId()) - .taskInstanceId(taskEventChangeCommand.getTaskInstanceId()) - .key(taskEventChangeCommand.getKey()) + .processInstanceId(taskInstanceWakeupRequest.getProcessInstanceId()) + .taskInstanceId(taskInstanceWakeupRequest.getTaskInstanceId()) + .key(taskInstanceWakeupRequest.getKey()) .type(StateEventType.WAKE_UP_TASK_GROUP) .build(); - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setWorkflowAndTaskInstanceIDMDC( - stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId())) { - log.info("Received task event change command, event: {}", stateEvent); + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId()); + log.info("Received wakeupTaskInstance request, event: {}", stateEvent); stateEventResponseService.addEvent2WorkflowExecute(stateEvent); + return TaskInstanceWakeupResponse.success(); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_WAKEUP_EVENT_REQUEST; - } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java new file mode 100644 index 0000000000..c3420cb1da --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.rpc; + +import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService; +import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto; +import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; +import org.apache.dolphinscheduler.server.master.service.ExecutingService; + +import java.util.Optional; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class WorkflowInstanceServiceImpl implements IWorkflowInstanceService { + + @Autowired + private ExecutingService executingService; + + @Override + public void clearWorkflowMetrics(Long workflowDefinitionCode) { + log.info("Receive clearWorkflowMetrics request: {}", workflowDefinitionCode); + ProcessInstanceMetrics.cleanUpProcessInstanceCountMetricsByDefinitionCode(workflowDefinitionCode); + } + + @Override + public WorkflowExecuteDto getWorkflowExecutingData(Integer workflowInstanceId) { + log.info("Receive getWorkflowExecutingData request: {}", workflowInstanceId); + Optional workflowExecuteDtoOptional = + executingService.queryWorkflowExecutingData(workflowInstanceId); + return workflowExecuteDtoOptional.orElse(null); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskDispatcher.java index 9878391ea5..0ad515e93a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/BaseTaskDispatcher.java @@ -21,20 +21,14 @@ import static com.google.common.base.Preconditions.checkNotNull; import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchRequest; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchResponse; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import org.apache.dolphinscheduler.server.master.exception.TaskDispatchException; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.dispatcher.TaskDispatcher; import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable; @@ -48,14 +42,11 @@ public abstract class BaseTaskDispatcher implements TaskDispatcher { protected final TaskEventService taskEventService; protected final MasterConfig masterConfig; - protected final MasterRpcClient masterRpcClient; protected BaseTaskDispatcher(TaskEventService taskEventService, - MasterConfig masterConfig, - MasterRpcClient masterRpcClient) { + MasterConfig masterConfig) { this.taskEventService = checkNotNull(taskEventService); this.masterConfig = checkNotNull(masterConfig); - this.masterRpcClient = checkNotNull(masterRpcClient); } @Override @@ -78,30 +69,9 @@ public abstract class BaseTaskDispatcher implements TaskDispatcher { addDispatchEvent(taskExecuteRunnable); } - protected abstract Optional getTaskInstanceDispatchHost(TaskExecuteRunnable taskExecutionContext) throws TaskDispatchException, WorkerGroupNotFoundException; + protected abstract void doDispatch(TaskExecuteRunnable taskExecuteRunnable) throws TaskDispatchException; - protected void doDispatch(TaskExecuteRunnable taskExecuteRunnable) throws TaskDispatchException { - TaskExecutionContext taskExecutionContext = taskExecuteRunnable.getTaskExecutionContext(); - TaskDispatchRequest taskDispatchRequest = new TaskDispatchRequest(taskExecutionContext); - try { - Message message = masterRpcClient.sendSyncCommand(Host.of(taskExecutionContext.getHost()), - taskDispatchRequest.convert2Command()); - TaskDispatchResponse taskDispatchResponse = - JSONUtils.parseObject(message.getBody(), TaskDispatchResponse.class); - if (!taskDispatchResponse.isDispatchSuccess()) { - throw new TaskDispatchException(String.format("Dispatch task to %s failed, response is: %s", - taskExecutionContext.getHost(), taskDispatchResponse)); - } - } catch (InterruptedException e) { - // This exception should only happen when we close the server. - Thread.currentThread().interrupt(); - throw new TaskDispatchException(String.format("Dispatch task to %s failed, get response failed", - taskExecutionContext.getHost()), e); - } catch (RemotingException e) { - throw new TaskDispatchException(String.format("Dispatch task to %s failed", - taskExecutionContext.getHost()), e); - } - } + protected abstract Optional getTaskInstanceDispatchHost(TaskExecuteRunnable taskExecutionContext) throws TaskDispatchException, WorkerGroupNotFoundException; protected void addDispatchEvent(TaskExecuteRunnable taskExecuteRunnable) { TaskExecutionContext taskExecutionContext = taskExecuteRunnable.getTaskExecutionContext(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java index 6ca37a3fe8..38086a8b46 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java @@ -130,9 +130,8 @@ public class StateWheelExecuteThread extends BaseDaemonThread { return; } for (Integer processInstanceId : processInstanceTimeoutCheckList) { - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowInstanceIdMDC(processInstanceId)) { + try { + LogUtils.setWorkflowInstanceIdMDC(processInstanceId); WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId( processInstanceId); if (workflowExecuteThread == null) { @@ -158,6 +157,8 @@ public class StateWheelExecuteThread extends BaseDaemonThread { } } catch (Exception ex) { log.error("Check workflow instance timeout error"); + } finally { + LogUtils.removeWorkflowInstanceIdMDC(); } } } @@ -220,9 +221,8 @@ public class StateWheelExecuteThread extends BaseDaemonThread { return; } for (TaskInstanceKey taskInstanceKey : taskInstanceTimeoutCheckList) { - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowInstanceIdMDC(taskInstanceKey.getProcessInstanceId())) { + try { + LogUtils.setWorkflowInstanceIdMDC(taskInstanceKey.getProcessInstanceId()); int processInstanceId = taskInstanceKey.getProcessInstanceId(); long taskCode = taskInstanceKey.getTaskCode(); @@ -257,6 +257,8 @@ public class StateWheelExecuteThread extends BaseDaemonThread { } } catch (Exception ex) { log.error("Check task timeout error, taskInstanceKey: {}", taskInstanceKey, ex); + } finally { + LogUtils.removeWorkflowInstanceIdMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java index 40ef9e5df6..a4b4f083d9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java @@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WOR import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskEventType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; @@ -31,6 +32,12 @@ import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionFinishEventAck; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionInfoEventAck; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; @@ -42,8 +49,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessageAck; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteStartMessage; import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; @@ -103,7 +108,7 @@ public class StreamTaskExecuteRunnable implements Runnable { protected ProcessDefinition processDefinition; - protected TaskExecuteStartMessage taskExecuteStartMessage; + protected StreamingTaskTriggerRequest taskExecuteStartMessage; protected TaskExecutionContextFactory taskExecutionContextFactory; @@ -114,7 +119,8 @@ public class StreamTaskExecuteRunnable implements Runnable { private TaskRunnableStatus taskRunnableStatus = TaskRunnableStatus.CREATED; - public StreamTaskExecuteRunnable(TaskDefinition taskDefinition, TaskExecuteStartMessage taskExecuteStartMessage) { + public StreamTaskExecuteRunnable(TaskDefinition taskDefinition, + StreamingTaskTriggerRequest taskExecuteStartMessage) { this.processService = SpringApplicationContext.getBean(ProcessService.class); this.masterConfig = SpringApplicationContext.getBean(MasterConfig.class); this.workerTaskDispatcher = SpringApplicationContext.getBean(WorkerTaskDispatcher.class); @@ -466,14 +472,27 @@ public class StreamTaskExecuteRunnable implements Runnable { private void sendAckToWorker(TaskEvent taskEvent) { // If event handle success, send ack to worker to otherwise the worker will retry this event - TaskExecuteRunningMessageAck taskExecuteRunningMessageAck = - new TaskExecuteRunningMessageAck( - true, - taskEvent.getTaskInstanceId(), - masterConfig.getMasterAddress(), - taskEvent.getWorkerAddress(), - System.currentTimeMillis()); - taskEvent.getChannel().writeAndFlush(taskExecuteRunningMessageAck.convert2Command()); + ITaskInstanceExecutionEventAckListener instanceExecutionEventAckListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskEvent.getWorkerAddress(), ITaskInstanceExecutionEventAckListener.class); + if (taskEvent.getEvent() == TaskEventType.RUNNING) { + log.error("taskEvent.getChannel() is null, taskEvent:{}", taskEvent); + instanceExecutionEventAckListener.handleTaskInstanceExecutionRunningEventAck( + TaskInstanceExecutionRunningEventAck.success(taskEvent.getTaskInstanceId())); + return; + } + if (taskEvent.getEvent() == TaskEventType.RESULT) { + instanceExecutionEventAckListener.handleTaskInstanceExecutionFinishEventAck( + TaskInstanceExecutionFinishEventAck.success(taskEvent.getTaskInstanceId())); + return; + } + + if (taskEvent.getEvent() == TaskEventType.UPDATE_PID) { + instanceExecutionEventAckListener.handleTaskInstanceExecutionInfoEventAck( + TaskInstanceExecutionInfoEventAck.success(taskEvent.getTaskInstanceId())); + return; + } + log.warn("SendAckToWorker error, get an unknown event: {}", taskEvent); } private enum TaskRunnableStatus { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java index af9eafbec3..b409327742 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java @@ -86,9 +86,8 @@ public class WorkflowEventLooper extends BaseDaemonThread implements AutoCloseab Thread.currentThread().interrupt(); break; } - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowInstanceIdMDC(workflowEvent.getWorkflowInstanceId())) { + try { + LogUtils.setWorkflowInstanceIdMDC(workflowEvent.getWorkflowInstanceId()); log.info("Begin to handle WorkflowEvent: {}", workflowEvent); WorkflowEventHandler workflowEventHandler = workflowEventHandlerMap.get(workflowEvent.getWorkflowEventType()); @@ -108,6 +107,8 @@ public class WorkflowEventLooper extends BaseDaemonThread implements AutoCloseab unknownException); workflowEventQueue.addEvent(workflowEvent); ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); + } finally { + LogUtils.removeWorkflowInstanceIdMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index e8e2a6a233..b3fc7c4960 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -57,18 +57,17 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.task.TaskWakeupRequest; -import org.apache.dolphinscheduler.remote.command.task.WorkflowHostChangeRequest; -import org.apache.dolphinscheduler.remote.command.task.WorkflowHostChangeResponse; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.StateEventHandleError; @@ -78,10 +77,8 @@ import org.apache.dolphinscheduler.server.master.event.StateEventHandler; import org.apache.dolphinscheduler.server.master.event.StateEventHandlerManager; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; -import org.apache.dolphinscheduler.server.master.exception.TaskExecuteRunnableCreateException; import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory; import org.apache.dolphinscheduler.server.master.utils.TaskUtils; @@ -143,8 +140,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { private final ProcessAlertManager processAlertManager; - private final MasterRpcClient masterRpcClient; - private final IWorkflowExecuteContext workflowExecuteContext; private WorkflowRunnableStatus workflowRunnableStatus = WorkflowRunnableStatus.CREATED; @@ -232,7 +227,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { @NonNull CommandService commandService, @NonNull ProcessService processService, @NonNull ProcessInstanceDao processInstanceDao, - @NonNull MasterRpcClient masterRpcClient, @NonNull ProcessAlertManager processAlertManager, @NonNull MasterConfig masterConfig, @NonNull StateWheelExecuteThread stateWheelExecuteThread, @@ -243,7 +237,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { this.commandService = commandService; this.processInstanceDao = processInstanceDao; this.workflowExecuteContext = workflowExecuteContext; - this.masterRpcClient = masterRpcClient; this.masterConfig = masterConfig; this.processAlertManager = processAlertManager; this.stateWheelExecuteThread = stateWheelExecuteThread; @@ -449,7 +442,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { * release task group * */ - public void releaseTaskGroup(TaskInstance taskInstance) throws RemotingException, InterruptedException { + public void releaseTaskGroup(TaskInstance taskInstance) throws InterruptedException { ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance(); // todo: use Integer if (taskInstance.getTaskGroupId() <= 0) { @@ -473,9 +466,10 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } else { ProcessInstance processInstance = processService.findProcessInstanceById(nextTaskInstance.getProcessInstanceId()); - masterRpcClient.sendSyncCommand( - Host.of(processInstance.getHost()), - new TaskWakeupRequest(processInstance.getId(), nextTaskInstance.getId()).convert2Command()); + ILogicTaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(processInstance.getHost(), ILogicTaskInstanceOperator.class); + taskInstanceOperator.wakeupTaskInstance( + new TaskInstanceWakeupRequest(processInstance.getId(), nextTaskInstance.getId())); } log.info("Success send wakeup message to next taskInstance: {}", nextTaskInstance.getId()); } @@ -829,9 +823,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { taskInstanceDao.queryValidTaskListByWorkflowInstanceId(workflowInstance.getId(), workflowInstance.getTestFlag()); for (TaskInstance task : validTaskInstanceList) { - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(task.getProcessInstanceId(), task.getId());) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(task.getProcessInstanceId(), task.getId()); log.info( "Check the taskInstance from a exist workflowInstance, existTaskInstanceCode: {}, taskInstanceStatus: {}", task.getTaskCode(), @@ -881,6 +874,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { if (task.getState().isFailure()) { errorTaskMap.put(task.getTaskCode(), task.getId()); } + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } clearDataIfExecuteTask(); @@ -947,9 +942,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { } // 2. create task execute runnable // in a dag, only one taskInstance is valid per taskCode, so need to set the old taskInstance invalid - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceIdMDC(taskInstance.getId())) { + try { + LogUtils.setTaskInstanceIdMDC(taskInstance.getId()); DefaultTaskExecuteRunnable taskExecuteRunnable = defaultTaskExecuteRunnableFactory.createTaskExecuteRunnable(taskInstance); if (validTaskMap.containsKey(taskInstance.getTaskCode())) { @@ -992,6 +986,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { stateWheelExecuteThread.addTask4TimeoutCheck(workflowInstance, taskInstance); return true; + } finally { + LogUtils.removeTaskInstanceIdMDC(); } } catch (Exception e) { log.error("Submit standby task {} error", taskInstance.getName(), e); @@ -1389,21 +1385,15 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { return false; } try { - Message message = - masterRpcClient.sendSyncCommand(Host.of(taskInstance.getHost()), - new WorkflowHostChangeRequest(taskInstance.getId(), masterConfig.getMasterAddress()) - .convert2Command()); - if (message == null) { + org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator iTaskInstanceOperator = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), + org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator.class); + UpdateWorkflowHostResponse updateWorkflowHostResponse = iTaskInstanceOperator.updateWorkflowInstanceHost( + new UpdateWorkflowHostRequest(taskInstance.getId(), masterConfig.getMasterAddress())); + if (!updateWorkflowHostResponse.isSuccess()) { log.error( - "Takeover task instance failed, the worker {} might not be alive, will try to create a new task instance", - taskInstance.getHost()); - return false; - } - WorkflowHostChangeResponse workflowHostChangeResponse = - JSONUtils.parseObject(message.getBody(), WorkflowHostChangeResponse.class); - if (workflowHostChangeResponse == null || !workflowHostChangeResponse.isSuccess()) { - log.error( - "Takeover task instance failed, receive a failed response from worker: {}, will try to create a new task instance", + "Takeover TaskInstance failed, receive a failed response from worker: {}, will try to create a new TaskInstance", taskInstance.getHost()); return false; } @@ -1417,9 +1407,9 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { stateWheelExecuteThread.addTask4TimeoutCheck(workflowInstance, taskInstance); stateWheelExecuteThread.addTask4RetryCheck(workflowInstance, taskInstance); return true; - } catch (RemotingException | InterruptedException | TaskExecuteRunnableCreateException e) { + } catch (Exception e) { log.error( - "Takeover task instance failed, the worker {} might not be alive, will try to create a new task instance", + "Takeover TaskInstance failed, the worker {} might not be alive, will try to create a new TaskInstance", taskInstance.getHost(), e); return false; } @@ -1872,9 +1862,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { if (taskInstanceId == null || taskInstanceId.equals(0)) { continue; } - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstance.getId(), taskInstanceId)) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstance.getId(), taskInstanceId); TaskInstance taskInstance = taskInstanceDao.queryById(taskInstanceId); if (taskInstance == null || taskInstance.getState().isFinished()) { continue; @@ -1890,6 +1879,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable { .build(); this.addStateEvent(taskStateEvent); } + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java index 0d04f2d92a..ed4fb8855b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.exception.WorkflowCreateException; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory; import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; import org.apache.dolphinscheduler.service.command.CommandService; @@ -49,9 +48,6 @@ public class WorkflowExecuteRunnableFactory { @Autowired private ProcessInstanceDao processInstanceDao; - @Autowired - private MasterRpcClient masterRpcClient; - @Autowired private ProcessAlertManager processAlertManager; @@ -82,7 +78,6 @@ public class WorkflowExecuteRunnableFactory { commandService, processService, processInstanceDao, - masterRpcClient, processAlertManager, masterConfig, stateWheelExecuteThread, diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java index 7911c338e0..8fe1a9a683 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java @@ -22,11 +22,11 @@ import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; @@ -67,9 +67,6 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor { @Autowired private ProcessInstanceExecCacheManager processInstanceExecCacheManager; - @Autowired - private StateEventCallbackService stateEventCallbackService; - @Autowired private StateWheelExecuteThread stateWheelExecuteThread; @@ -171,9 +168,8 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor { TaskInstance taskInstance = entry.getValue(); crossWorkflowParameterPassing(finishProcessInstance, taskInstance); String address = NetUtils.getAddr(masterConfig.getListenPort()); - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(processInstance.getId(), taskInstance.getId())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(processInstance.getId(), taskInstance.getId()); if (processInstance.getHost().equalsIgnoreCase(address)) { log.info("Process host is local master, will notify it"); this.notifyMyself(processInstance, taskInstance); @@ -181,6 +177,8 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor { log.info("Process host is remote master, will notify it"); this.notifyProcess(finishProcessInstance, processInstance, taskInstance); } + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } } @@ -227,10 +225,13 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor { taskInstance.getName(), taskInstance.getId()); return; } - WorkflowStateEventChangeRequest workflowStateEventChangeRequest = new WorkflowStateEventChangeRequest( + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(processInstanceHost, ITaskInstanceExecutionEventListener.class); + + WorkflowInstanceStateChangeEvent workflowInstanceStateChangeEvent = new WorkflowInstanceStateChangeEvent( finishProcessInstance.getId(), 0, finishProcessInstance.getState(), processInstance.getId(), taskInstance.getId()); - Host host = new Host(processInstanceHost); - stateEventCallbackService.sendResult(host, workflowStateEventChangeRequest.convert2Command()); + iTaskInstanceExecutionEventListener.onWorkflowInstanceInstanceStateChange(workflowInstanceStateChangeEvent); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java index f4fb3a908d..4a4ca42c58 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java @@ -17,10 +17,15 @@ package org.apache.dolphinscheduler.server.master.runner.dispatcher; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.base.utils.Host; +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.exception.TaskDispatchException; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.BaseTaskDispatcher; import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable; @@ -37,12 +42,31 @@ public class MasterTaskDispatcher extends BaseTaskDispatcher { private final Optional masterTaskExecuteHost; public MasterTaskDispatcher(TaskEventService taskEventService, - MasterConfig masterConfig, - MasterRpcClient masterRpcClient) { - super(taskEventService, masterConfig, masterRpcClient); + MasterConfig masterConfig) { + super(taskEventService, masterConfig); masterTaskExecuteHost = Optional.of(Host.of(masterConfig.getMasterAddress())); } + @Override + protected void doDispatch(TaskExecuteRunnable taskExecuteRunnable) throws TaskDispatchException { + TaskExecutionContext taskExecutionContext = taskExecuteRunnable.getTaskExecutionContext(); + try { + ILogicTaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskExecutionContext.getHost(), ILogicTaskInstanceOperator.class); + LogicTaskDispatchResponse logicTaskDispatchResponse = taskInstanceOperator + .dispatchLogicTask(new LogicTaskDispatchRequest(taskExecuteRunnable.getTaskExecutionContext())); + if (!logicTaskDispatchResponse.isDispatchSuccess()) { + throw new TaskDispatchException(String.format("Dispatch LogicTask to %s failed, response is: %s", + taskExecutionContext.getHost(), logicTaskDispatchResponse)); + } + } catch (TaskDispatchException e) { + throw e; + } catch (Exception e) { + throw new TaskDispatchException(String.format("Dispatch task to %s failed", + taskExecutionContext.getHost()), e); + } + } + @Override protected Optional getTaskInstanceDispatchHost(TaskExecuteRunnable taskExecutionContext) { return masterTaskExecuteHost; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java index b2c256cd26..6760633b9b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java @@ -19,12 +19,17 @@ package org.apache.dolphinscheduler.server.master.runner.dispatcher; import static com.google.common.base.Preconditions.checkNotNull; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.base.utils.Host; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchResponse; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager; +import org.apache.dolphinscheduler.server.master.exception.TaskDispatchException; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.BaseTaskDispatcher; import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable; @@ -42,12 +47,31 @@ public class WorkerTaskDispatcher extends BaseTaskDispatcher { public WorkerTaskDispatcher(TaskEventService taskEventService, MasterConfig masterConfig, - MasterRpcClient masterRpcClient, HostManager hostManager) { - super(taskEventService, masterConfig, masterRpcClient); + super(taskEventService, masterConfig); this.hostManager = checkNotNull(hostManager); } + @Override + protected void doDispatch(TaskExecuteRunnable taskExecuteRunnable) throws TaskDispatchException { + TaskExecutionContext taskExecutionContext = taskExecuteRunnable.getTaskExecutionContext(); + try { + ITaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskExecutionContext.getHost(), ITaskInstanceOperator.class); + TaskInstanceDispatchResponse taskInstanceDispatchResponse = taskInstanceOperator + .dispatchTask(new TaskInstanceDispatchRequest(taskExecuteRunnable.getTaskExecutionContext())); + if (!taskInstanceDispatchResponse.isDispatchSuccess()) { + throw new TaskDispatchException(String.format("Dispatch task to %s failed, response is: %s", + taskExecutionContext.getHost(), taskInstanceDispatchResponse)); + } + } catch (TaskDispatchException e) { + throw e; + } catch (Exception e) { + throw new TaskDispatchException(String.format("Dispatch task to %s failed", + taskExecutionContext.getHost()), e); + } + } + @Override protected Optional getTaskInstanceDispatchHost(TaskExecuteRunnable taskExecuteRunnable) throws WorkerGroupNotFoundException { String workerGroup = taskExecuteRunnable.getTaskExecutionContext().getWorkerGroup(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnable.java index 5b41481714..8a6e6d8e87 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnable.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.master.runner.execute; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; +import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; import org.apache.dolphinscheduler.server.master.runner.task.IAsyncLogicTask; import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; @@ -32,9 +32,9 @@ public class AsyncMasterDelayTaskExecuteRunnable extends MasterDelayTaskExecuteR public AsyncMasterDelayTaskExecuteRunnable(TaskExecutionContext taskExecutionContext, LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder, - MasterMessageSenderManager masterMessageSenderManager, + LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager, AsyncMasterTaskDelayQueue asyncTaskDelayQueue) { - super(taskExecutionContext, logicTaskPluginFactoryBuilder, masterMessageSenderManager); + super(taskExecutionContext, logicTaskPluginFactoryBuilder, logicTaskInstanceExecutionEventSenderManager); this.asyncMasterTaskDelayQueue = asyncTaskDelayQueue; } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnableFactory.java index 169d57a2e4..a71f394b7d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterDelayTaskExecuteRunnableFactory.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.master.runner.execute; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; +import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +33,7 @@ public class AsyncMasterDelayTaskExecuteRunnableFactory private LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder; @Autowired - private MasterMessageSenderManager masterMessageSenderManager; + private LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager; @Autowired private AsyncMasterTaskDelayQueue asyncTaskDelayQueue; @@ -42,7 +42,7 @@ public class AsyncMasterDelayTaskExecuteRunnableFactory public AsyncMasterDelayTaskExecuteRunnable createWorkerTaskExecuteRunnable(TaskExecutionContext taskExecutionContext) { return new AsyncMasterDelayTaskExecuteRunnable(taskExecutionContext, logicTaskPluginFactoryBuilder, - masterMessageSenderManager, + logicTaskInstanceExecutionEventSenderManager, asyncTaskDelayQueue); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java index afdab35eb1..63b8636be6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java @@ -75,11 +75,10 @@ public class AsyncMasterTaskDelayQueueLooper extends BaseDaemonThread implements break; } final TaskExecutionContext taskExecutionContext = asyncTaskExecutionContext.getTaskExecutionContext(); - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setWorkflowAndTaskInstanceIDMDC( - taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); - LogUtils.MDCAutoClosableContext mdcAutoClosableContext1 = - LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); + LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath()); if (MasterTaskExecutionContextHolder .getTaskExecutionContext(taskExecutionContext.getTaskInstanceId()) == null) { @@ -118,6 +117,9 @@ public class AsyncMasterTaskDelayQueueLooper extends BaseDaemonThread implements LogUtils.removeTaskInstanceIdMDC(); } }); + } finally { + LogUtils.removeTaskInstanceLogFullPathMDC(); + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } log.info("AsyncMasterTaskDelayQueueLooper closed..."); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnable.java index 4e9a5aed9f..6f736139b6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnable.java @@ -22,37 +22,37 @@ import static com.google.common.base.Preconditions.checkNotNull; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.runner.operator.TaskOperatorManager; +import org.apache.dolphinscheduler.server.master.runner.operator.TaskExecuteRunnableOperatorManager; public class DefaultTaskExecuteRunnable extends PriorityTaskExecuteRunnable { - private final TaskOperatorManager taskOperatorManager; + private final TaskExecuteRunnableOperatorManager taskExecuteRunnableOperatorManager; public DefaultTaskExecuteRunnable(ProcessInstance workflowInstance, TaskInstance taskInstance, TaskExecutionContext taskExecutionContext, - TaskOperatorManager taskOperatorManager) { + TaskExecuteRunnableOperatorManager taskExecuteRunnableOperatorManager) { super(workflowInstance, taskInstance, taskExecutionContext); - this.taskOperatorManager = checkNotNull(taskOperatorManager); + this.taskExecuteRunnableOperatorManager = checkNotNull(taskExecuteRunnableOperatorManager); } @Override public void dispatch() { - taskOperatorManager.getTaskDispatchOperator().handle(this); + taskExecuteRunnableOperatorManager.getTaskDispatchOperator(this).operate(this); } @Override public void kill() { - taskOperatorManager.getTaskKillOperator().handle(this); + taskExecuteRunnableOperatorManager.getTaskKillOperator(this).operate(this); } @Override public void pause() { - taskOperatorManager.getTaskPauseOperator().handle(this); + taskExecuteRunnableOperatorManager.getTaskPauseOperator(this).operate(this); } @Override public void timeout() { - taskOperatorManager.getTaskTimeoutOperator().handle(this); + taskExecuteRunnableOperatorManager.getTaskTimeoutOperator(this).operate(this); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnableFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnableFactory.java index 7c447ba02b..64443855fa 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/DefaultTaskExecuteRunnableFactory.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheM import org.apache.dolphinscheduler.server.master.exception.TaskExecuteRunnableCreateException; import org.apache.dolphinscheduler.server.master.exception.TaskExecutionContextCreateException; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.operator.TaskOperatorManager; +import org.apache.dolphinscheduler.server.master.runner.operator.TaskExecuteRunnableOperatorManager; import lombok.extern.slf4j.Slf4j; @@ -40,7 +40,7 @@ public class DefaultTaskExecuteRunnableFactory implements TaskExecuteRunnableFac private TaskExecutionContextFactory taskExecutionContextFactory; @Autowired - private TaskOperatorManager taskOperatorManager; + private TaskExecuteRunnableOperatorManager taskExecuteRunnableOperatorManager; @Override public DefaultTaskExecuteRunnable createTaskExecuteRunnable(TaskInstance taskInstance) throws TaskExecuteRunnableCreateException { @@ -51,7 +51,7 @@ public class DefaultTaskExecuteRunnableFactory implements TaskExecuteRunnableFac workflowExecuteRunnable.getWorkflowExecuteContext().getWorkflowInstance(), taskInstance, taskExecutionContextFactory.createTaskExecutionContext(taskInstance), - taskOperatorManager); + taskExecuteRunnableOperatorManager); } catch (TaskExecutionContextCreateException ex) { throw new TaskExecuteRunnableCreateException("Create DefaultTaskExecuteRunnable failed", ex); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterDelayTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterDelayTaskExecuteRunnable.java index 653901f4de..70f3e93521 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterDelayTaskExecuteRunnable.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.master.runner.execute; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; +import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; import java.util.concurrent.Delayed; @@ -29,8 +29,8 @@ public abstract class MasterDelayTaskExecuteRunnable extends MasterTaskExecuteRu public MasterDelayTaskExecuteRunnable(TaskExecutionContext taskExecutionContext, LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder, - MasterMessageSenderManager masterMessageSenderManager) { - super(taskExecutionContext, logicTaskPluginFactoryBuilder, masterMessageSenderManager); + LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager) { + super(taskExecutionContext, logicTaskPluginFactoryBuilder, logicTaskInstanceExecutionEventSenderManager); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecuteRunnable.java index 8abcebc48e..5fa1364616 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecuteRunnable.java @@ -25,11 +25,10 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.server.master.exception.LogicTaskFactoryNotFoundException; import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; +import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTask; import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; @@ -40,15 +39,15 @@ public abstract class MasterTaskExecuteRunnable implements Runnable { protected final TaskExecutionContext taskExecutionContext; protected final LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder; - protected final MasterMessageSenderManager masterMessageSenderManager; + protected final LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager; protected ILogicTask logicTask; public MasterTaskExecuteRunnable(TaskExecutionContext taskExecutionContext, LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder, - MasterMessageSenderManager masterMessageSenderManager) { + LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager) { this.taskExecutionContext = taskExecutionContext; this.logicTaskPluginFactoryBuilder = logicTaskPluginFactoryBuilder; - this.masterMessageSenderManager = masterMessageSenderManager; + this.logicTaskInstanceExecutionEventSenderManager = logicTaskInstanceExecutionEventSenderManager; } protected abstract void executeTask() throws MasterTaskExecuteException; @@ -134,9 +133,10 @@ public abstract class MasterTaskExecuteRunnable implements Runnable { log.info("End initialize task {}", JSONUtils.toPrettyJsonString(taskExecutionContext)); } - protected void beforeExecute() throws LogicTaskFactoryNotFoundException, LogicTaskInitializeException, RemotingException { + protected void beforeExecute() throws LogicTaskFactoryNotFoundException, LogicTaskInitializeException { taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.RUNNING_EXECUTION); - masterMessageSenderManager.getMasterTaskExecuteRunningMessageSender().sendMessage(taskExecutionContext); + logicTaskInstanceExecutionEventSenderManager.getMasterTaskExecuteRunningMessageSender() + .sendMessage(taskExecutionContext); log.info("Send task status {} to master {}", taskExecutionContext.getCurrentExecutionStatus().name(), taskExecutionContext.getWorkflowInstanceHost()); @@ -166,7 +166,8 @@ public abstract class MasterTaskExecuteRunnable implements Runnable { try { taskExecutionContext.setEndTime(System.currentTimeMillis()); taskExecutionContext.setVarPool(JSONUtils.toJsonString(logicTask.getTaskParameters().getVarPool())); - masterMessageSenderManager.getMasterTaskExecuteResultMessageSender().sendMessage(taskExecutionContext); + logicTaskInstanceExecutionEventSenderManager.getLogicTaskInstanceExecutionFinishEventSender() + .sendMessage(taskExecutionContext); log.info("Send task status: {} to master: {} successfully", taskExecutionContext.getCurrentExecutionStatus().name(), taskExecutionContext.getWorkflowInstanceHost()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnable.java index 04f9321c08..62365f78cb 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnable.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.server.master.runner.execute; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; +import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; import org.apache.dolphinscheduler.server.master.runner.task.ISyncLogicTask; import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; @@ -31,8 +31,8 @@ public class SyncMasterDelayTaskExecuteRunnable extends MasterDelayTaskExecuteRu public SyncMasterDelayTaskExecuteRunnable(TaskExecutionContext taskExecutionContext, LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder, - MasterMessageSenderManager masterMessageSenderManager) { - super(taskExecutionContext, logicTaskPluginFactoryBuilder, masterMessageSenderManager); + LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager) { + super(taskExecutionContext, logicTaskPluginFactoryBuilder, logicTaskInstanceExecutionEventSenderManager); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnableFactory.java index 8c254485ae..2bf829c021 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterDelayTaskExecuteRunnableFactory.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.master.runner.execute; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.runner.message.MasterMessageSenderManager; +import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager; import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder; import lombok.extern.slf4j.Slf4j; @@ -35,11 +35,11 @@ public class SyncMasterDelayTaskExecuteRunnableFactory @Autowired private LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder; @Autowired - private MasterMessageSenderManager masterMessageSenderManager; + private LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager; @Override public SyncMasterDelayTaskExecuteRunnable createWorkerTaskExecuteRunnable(TaskExecutionContext taskExecutionContext) { return new SyncMasterDelayTaskExecuteRunnable(taskExecutionContext, logicTaskPluginFactoryBuilder, - masterMessageSenderManager); + logicTaskInstanceExecutionEventSenderManager); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/ITaskInstanceExecutionEventListenFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/ITaskInstanceExecutionEventListenFunction.java new file mode 100644 index 0000000000..5be6d207a0 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/ITaskInstanceExecutionEventListenFunction.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.listener; + +public interface ITaskInstanceExecutionEventListenFunction { + + void handleTaskInstanceExecutionEvent(E e); + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionEventListenerFunctionManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionEventListenerFunctionManager.java new file mode 100644 index 0000000000..239cc6def0 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionEventListenerFunctionManager.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.listener; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionEventListenerFunctionManager { + + @Autowired + private TaskInstanceExecutionRunningEventListenFunction taskInstanceExecutionRunningEventListenFunction; + + @Autowired + private TaskInstanceExecutionResultEventListenFunction taskInstanceExecutionResultEventListenFunction; + + @Autowired + private TaskInstanceExecutionInfoEventListenFunction taskInstanceExecutionInfoEventListenFunction; + + @Autowired + private TaskInstanceStateEventListenFunction taskInstanceStateEventListenFunction; + + public TaskInstanceExecutionRunningEventListenFunction getTaskInstanceExecutionRunningEventListenFunction() { + return taskInstanceExecutionRunningEventListenFunction; + } + + public TaskInstanceExecutionResultEventListenFunction getTaskInstanceExecutionResultEventListenFunction() { + return taskInstanceExecutionResultEventListenFunction; + } + + public TaskInstanceExecutionInfoEventListenFunction getTaskInstanceExecutionInfoEventListenFunction() { + return taskInstanceExecutionInfoEventListenFunction; + } + + public TaskInstanceStateEventListenFunction getTaskInstanceStateEventListenFunction() { + return taskInstanceStateEventListenFunction; + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskOperatorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionInfoEventListenFunction.java similarity index 53% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskOperatorManager.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionInfoEventListenFunction.java index fd647240e0..47dec6ba85 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskOperatorManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionInfoEventListenFunction.java @@ -15,40 +15,29 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.runner.operator; +package org.apache.dolphinscheduler.server.master.runner.listener; + +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionInfoEvent; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; + +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +@Slf4j @Component -public class TaskOperatorManager { - - @Autowired - private TaskKillOperator taskKillOperator; - - @Autowired - private TaskPauseOperator taskPauseOperator; - - @Autowired - private TaskDispatchOperator taskDispatchOperator; +public class TaskInstanceExecutionInfoEventListenFunction + implements + ITaskInstanceExecutionEventListenFunction { @Autowired - private TaskTimeoutOperator taskTimeoutOperator; + private TaskEventService taskEventService; - public TaskOperator getTaskKillOperator() { - return taskKillOperator; + @Override + public void handleTaskInstanceExecutionEvent(TaskInstanceExecutionInfoEvent taskInstanceExecutionInfoEvent) { + TaskEvent taskEvent = TaskEvent.newUpdatePidEvent(taskInstanceExecutionInfoEvent); + taskEventService.addEvent(taskEvent); } - - public TaskPauseOperator getTaskPauseOperator() { - return taskPauseOperator; - } - - public TaskDispatchOperator getTaskDispatchOperator() { - return taskDispatchOperator; - } - - public TaskTimeoutOperator getTaskTimeoutOperator() { - return taskTimeoutOperator; - } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionResultEventListenFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionResultEventListenFunction.java new file mode 100644 index 0000000000..45766f1697 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionResultEventListenFunction.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.listener; + +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionResultEventListenFunction + implements + ITaskInstanceExecutionEventListenFunction { + + @Autowired + private TaskEventService taskEventService; + + @Override + public void handleTaskInstanceExecutionEvent(TaskInstanceExecutionFinishEvent taskInstanceExecutionFinishEvent) { + TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskInstanceExecutionFinishEvent); + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(taskResultEvent.getProcessInstanceId(), + taskResultEvent.getTaskInstanceId()); + log.info("Received TaskInstanceExecutionFinishEvent: {}", taskResultEvent); + taskEventService.addEvent(taskResultEvent); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); + } + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionRunningEventListenFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionRunningEventListenFunction.java new file mode 100644 index 0000000000..5cdd39c3fd --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceExecutionRunningEventListenFunction.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.listener; + +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionRunningEvent; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; +import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionRunningEventListenFunction + implements + ITaskInstanceExecutionEventListenFunction { + + @Autowired + private TaskEventService taskEventService; + + @Override + public void handleTaskInstanceExecutionEvent(TaskInstanceExecutionRunningEvent taskInstanceExecutionRunningEvent) { + log.info("Receive handleTaskInstanceExecutionEvent request: {}", taskInstanceExecutionRunningEvent); + taskEventService.addEvent(TaskEvent.newRunningEvent(taskInstanceExecutionRunningEvent)); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceStateEventListenFunction.java similarity index 64% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceStateEventListenFunction.java index bfee81dc48..1beea02443 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/listener/TaskInstanceStateEventListenFunction.java @@ -15,16 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.processor; +package org.apache.dolphinscheduler.server.master.runner.listener; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.processor.MasterRpcProcessor; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; @@ -35,44 +31,34 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.netty.channel.Channel; - -/** - * handle state event received from master/api - */ -@Component @Slf4j -public class StateEventProcessor implements MasterRpcProcessor { +@Component +public class TaskInstanceStateEventListenFunction + implements + ITaskInstanceExecutionEventListenFunction { @Autowired private StateEventResponseService stateEventResponseService; @Override - public void process(Channel channel, Message message) { - WorkflowStateEventChangeRequest workflowStateEventChangeRequest = - JSONUtils.parseObject(message.getBody(), WorkflowStateEventChangeRequest.class); - StateEvent stateEvent; - if (workflowStateEventChangeRequest.getDestTaskInstanceId() == 0) { - stateEvent = createWorkflowStateEvent(workflowStateEventChangeRequest); + public void handleTaskInstanceExecutionEvent(WorkflowInstanceStateChangeEvent taskInstanceInstanceStateChangeEvent) { + final StateEvent stateEvent; + if (taskInstanceInstanceStateChangeEvent.getDestTaskInstanceId() == 0) { + stateEvent = createWorkflowStateEvent(taskInstanceInstanceStateChangeEvent); } else { - stateEvent = createTaskStateEvent(workflowStateEventChangeRequest); + stateEvent = createTaskStateEvent(taskInstanceInstanceStateChangeEvent); } - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setWorkflowAndTaskInstanceIDMDC( - stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId()); log.info("Received state change command, event: {}", stateEvent); stateEventResponseService.addStateChangeEvent(stateEvent); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } - } - @Override - public MessageType getCommandType() { - return MessageType.STATE_EVENT_REQUEST; - } - - private TaskStateEvent createTaskStateEvent(WorkflowStateEventChangeRequest workflowStateEventChangeRequest) { + private TaskStateEvent createTaskStateEvent(WorkflowInstanceStateChangeEvent workflowStateEventChangeRequest) { return TaskStateEvent.builder() .processInstanceId(workflowStateEventChangeRequest.getDestProcessInstanceId()) .taskInstanceId(workflowStateEventChangeRequest.getDestTaskInstanceId()) @@ -81,7 +67,7 @@ public class StateEventProcessor implements MasterRpcProcessor { .build(); } - private WorkflowStateEvent createWorkflowStateEvent(WorkflowStateEventChangeRequest workflowStateEventChangeRequest) { + private WorkflowStateEvent createWorkflowStateEvent(WorkflowInstanceStateChangeEvent workflowStateEventChangeRequest) { WorkflowExecutionStatus workflowExecutionStatus = workflowStateEventChangeRequest.getSourceStatus(); if (workflowStateEventChangeRequest.getSourceProcessInstanceId() != workflowStateEventChangeRequest .getDestProcessInstanceId()) { @@ -94,5 +80,4 @@ public class StateEventProcessor implements MasterRpcProcessor { .key(workflowStateEventChangeRequest.getKey()) .build(); } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskExecuteRunningMessageSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteRunningEventSender.java similarity index 56% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskExecuteRunningMessageSender.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteRunningEventSender.java index ca846bbb57..1dcca72878 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskExecuteRunningMessageSender.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteRunningEventSender.java @@ -17,39 +17,32 @@ package org.apache.dolphinscheduler.server.master.runner.message; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionRunningEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessage; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import lombok.NonNull; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class MasterTaskExecuteRunningMessageSender implements MasterMessageSender { - - @Autowired - private MasterRpcClient masterRpcClient; - - @Autowired - private MasterConfig masterConfig; +public class LogicTaskInstanceExecuteRunningEventSender + implements + LogicTaskInstanceExecutionEventSender { @Override - public void sendMessage(TaskExecuteRunningMessage message) throws RemotingException { - masterRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command()); + public void sendMessage(TaskInstanceExecutionRunningEvent taskInstanceExecutionRunningEvent) { + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstanceExecutionRunningEvent.getHost(), + ITaskInstanceExecutionEventListener.class); + iTaskInstanceExecutionEventListener.onTaskInstanceExecutionRunning(taskInstanceExecutionRunningEvent); } @Override - public TaskExecuteRunningMessage buildMessage(@NonNull TaskExecutionContext taskExecutionContext) { - TaskExecuteRunningMessage taskExecuteRunningMessage = - new TaskExecuteRunningMessage(masterConfig.getMasterAddress(), - taskExecutionContext.getWorkflowInstanceHost(), - System.currentTimeMillis()); + public TaskInstanceExecutionRunningEvent buildMessage(@NonNull TaskExecutionContext taskExecutionContext) { + TaskInstanceExecutionRunningEvent taskExecuteRunningMessage = new TaskInstanceExecutionRunningEvent(); taskExecuteRunningMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskExecuteRunningMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskExecuteRunningMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus()); @@ -61,8 +54,4 @@ public class MasterTaskExecuteRunningMessageSender implements MasterMessageSende return taskExecuteRunningMessage; } - @Override - public MessageType getMessageType() { - return MessageType.TASK_EXECUTE_RUNNING_MESSAGE; - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterMessageSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSender.java similarity index 71% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterMessageSender.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSender.java index 8c2693b0c6..3b3ff7f645 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterMessageSender.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSender.java @@ -18,21 +18,16 @@ package org.apache.dolphinscheduler.server.master.runner.message; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; // todo: unify with WorkerMessageSender -public interface MasterMessageSender { +public interface LogicTaskInstanceExecutionEventSender { /** * Send the message - * - * @throws RemotingException Cannot connect to the target host. */ - void sendMessage(T message) throws RemotingException; + void sendMessage(T message); - default void sendMessage(TaskExecutionContext taskExecutionContext) throws RemotingException { + default void sendMessage(TaskExecutionContext taskExecutionContext) { T message = buildMessage(taskExecutionContext); sendMessage(message); } @@ -42,9 +37,4 @@ public interface MasterMessageSender { */ T buildMessage(TaskExecutionContext taskExecutionContext); - /** - * The message type can be sent by this sender. - */ - MessageType getMessageType(); - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterMessageSenderManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSenderManager.java similarity index 58% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterMessageSenderManager.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSenderManager.java index 82fbb00c41..1625dd73b1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterMessageSenderManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSenderManager.java @@ -24,27 +24,20 @@ import org.springframework.stereotype.Component; @Slf4j @Component -public class MasterMessageSenderManager { +public class LogicTaskInstanceExecutionEventSenderManager { @Autowired - private MasterTaskExecuteResultMessageSender masterTaskExecuteResultMessageSender; + private LogicTaskInstanceExecutionFinishEventSender logicTaskInstanceExecutionFinishEventSender; @Autowired - private MasterTaskExecuteRunningMessageSender masterTaskExecuteRunningMessageSender; + private LogicTaskInstanceExecuteRunningEventSender logicTaskInstanceExecuteRunningEventSender; - @Autowired - private MasterTaskUpdateRuntimeMessageSender masterTaskUpdateRuntimeMessageSender; - - public MasterTaskExecuteResultMessageSender getMasterTaskExecuteResultMessageSender() { - return masterTaskExecuteResultMessageSender; - } - - public MasterTaskExecuteRunningMessageSender getMasterTaskExecuteRunningMessageSender() { - return masterTaskExecuteRunningMessageSender; + public LogicTaskInstanceExecutionFinishEventSender getLogicTaskInstanceExecutionFinishEventSender() { + return logicTaskInstanceExecutionFinishEventSender; } - public MasterTaskUpdateRuntimeMessageSender getMasterTaskUpdatePidMessageSender() { - return masterTaskUpdateRuntimeMessageSender; + public LogicTaskInstanceExecuteRunningEventSender getMasterTaskExecuteRunningMessageSender() { + return logicTaskInstanceExecuteRunningEventSender; } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskExecuteResultMessageSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionFinishEventSender.java similarity index 60% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskExecuteResultMessageSender.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionFinishEventSender.java index 90986a59eb..94fc1f57ec 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskExecuteResultMessageSender.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionFinishEventSender.java @@ -17,37 +17,29 @@ package org.apache.dolphinscheduler.server.master.runner.message; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessage; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class MasterTaskExecuteResultMessageSender implements MasterMessageSender { - - @Autowired - private MasterConfig masterConfig; - - @Autowired - private MasterRpcClient masterRpcClient; +public class LogicTaskInstanceExecutionFinishEventSender + implements + LogicTaskInstanceExecutionEventSender { @Override - public void sendMessage(TaskExecuteResultMessage message) throws RemotingException { - masterRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command()); + public void sendMessage(TaskInstanceExecutionFinishEvent message) { + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(message.getHost(), ITaskInstanceExecutionEventListener.class); + iTaskInstanceExecutionEventListener.onTaskInstanceExecutionFinish(new TaskInstanceExecutionFinishEvent()); } @Override - public TaskExecuteResultMessage buildMessage(TaskExecutionContext taskExecutionContext) { - TaskExecuteResultMessage taskExecuteResultMessage = - new TaskExecuteResultMessage(masterConfig.getMasterAddress(), - taskExecutionContext.getWorkflowInstanceHost(), - System.currentTimeMillis()); + public TaskInstanceExecutionFinishEvent buildMessage(TaskExecutionContext taskExecutionContext) { + TaskInstanceExecutionFinishEvent taskExecuteResultMessage = new TaskInstanceExecutionFinishEvent(); taskExecuteResultMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskExecuteResultMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskExecuteResultMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); @@ -63,8 +55,4 @@ public class MasterTaskExecuteResultMessageSender implements MasterMessageSender return taskExecuteResultMessage; } - @Override - public MessageType getMessageType() { - return MessageType.TASK_EXECUTE_RESULT_MESSAGE; - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskUpdateRuntimeMessageSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskUpdateRuntimeMessageSender.java deleted file mode 100644 index 6af8f3f391..0000000000 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/MasterTaskUpdateRuntimeMessageSender.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.runner.message; - -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskUpdateRuntimeMessage; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; - -import lombok.NonNull; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class MasterTaskUpdateRuntimeMessageSender implements MasterMessageSender { - - @Autowired - private MasterRpcClient masterRpcClient; - - @Autowired - private MasterConfig masterConfig; - - @Override - public void sendMessage(TaskUpdateRuntimeMessage message) throws RemotingException { - masterRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command()); - } - - @Override - public TaskUpdateRuntimeMessage buildMessage(@NonNull TaskExecutionContext taskExecutionContext) { - TaskUpdateRuntimeMessage taskUpdatePidRequest = - new TaskUpdateRuntimeMessage(masterConfig.getMasterAddress(), - taskExecutionContext.getWorkflowInstanceHost(), - System.currentTimeMillis()); - taskUpdatePidRequest.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - taskUpdatePidRequest.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); - taskUpdatePidRequest.setHost(taskExecutionContext.getHost()); - taskUpdatePidRequest.setStartTime(taskExecutionContext.getStartTime()); - return taskUpdatePidRequest; - } - - @Override - public MessageType getMessageType() { - return MessageType.TASK_UPDATE_RUNTIME_MESSAGE; - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java new file mode 100644 index 0000000000..6294f581ec --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableDispatchOperator.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.server.master.runner.GlobalTaskDispatchWaitingQueue; +import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; + +public abstract class BaseTaskExecuteRunnableDispatchOperator implements TaskExecuteRunnableOperator { + + private final GlobalTaskDispatchWaitingQueue globalTaskDispatchWaitingQueue; + + public BaseTaskExecuteRunnableDispatchOperator(GlobalTaskDispatchWaitingQueue globalTaskDispatchWaitingQueue) { + this.globalTaskDispatchWaitingQueue = globalTaskDispatchWaitingQueue; + } + + @Override + public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { + globalTaskDispatchWaitingQueue.submitNeedToDispatchTaskExecuteRunnable(taskExecuteRunnable); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskKillOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableKillOperator.java similarity index 66% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskKillOperator.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableKillOperator.java index c2bf2867c5..a1d0a893fc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskKillOperator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableKillOperator.java @@ -20,33 +20,23 @@ package org.apache.dolphinscheduler.server.master.runner.operator; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.task.TaskKillRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; -import org.apache.commons.lang3.StringUtils; - import java.util.Date; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - @Slf4j -@Component -public class TaskKillOperator implements TaskOperator { +public abstract class BaseTaskExecuteRunnableKillOperator implements TaskExecuteRunnableOperator { - @Autowired - private TaskInstanceDao taskInstanceDao; + private final TaskInstanceDao taskInstanceDao; - @Autowired - private MasterRpcClient masterRpcClient; + public BaseTaskExecuteRunnableKillOperator(TaskInstanceDao taskInstanceDao) { + this.taskInstanceDao = taskInstanceDao; + } @Override - public void handle(DefaultTaskExecuteRunnable taskExecuteRunnable) { + public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { TaskInstance taskInstance = taskExecuteRunnable.getTaskInstance(); log.info("Begin to kill task instance: {}", taskInstance.getName()); if (taskInstance.getState().isFinished()) { @@ -62,18 +52,11 @@ public class TaskKillOperator implements TaskOperator { } } + protected abstract void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance); + private void killTaskInstanceInDB(TaskInstance taskInstance) { taskInstance.setState(TaskExecutionStatus.KILL); taskInstance.setEndTime(new Date()); taskInstanceDao.updateById(taskInstance); } - - private void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) throws RemotingException { - if (StringUtils.isEmpty(taskInstance.getHost())) { - return; - } - TaskKillRequest killCommand = new TaskKillRequest(taskInstance.getId()); - masterRpcClient.send(Host.of(taskInstance.getHost()), killCommand.convert2Command()); - } - } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnablePauseOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnablePauseOperator.java new file mode 100644 index 0000000000..383752d3f3 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnablePauseOperator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class BaseTaskExecuteRunnablePauseOperator implements TaskExecuteRunnableOperator { + + @Override + public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { + try { + pauseRemoteTaskInstanceInThreadPool(taskExecuteRunnable.getTaskInstance()); + } catch (Exception e) { + log.error("Pause DefaultTaskExecuteRunnable failed", e); + } + } + + protected abstract void pauseRemoteTaskInstanceInThreadPool(TaskInstance taskInstance); +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskTimeoutOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableTimeoutOperator.java similarity index 74% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskTimeoutOperator.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableTimeoutOperator.java index 0da88a0dd3..4068bfa082 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskTimeoutOperator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/BaseTaskExecuteRunnableTimeoutOperator.java @@ -21,33 +21,23 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; -import org.apache.dolphinscheduler.remote.command.task.TaskKillRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; -import org.apache.commons.lang3.StringUtils; - import java.util.Date; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - @Slf4j -@Component -public class TaskTimeoutOperator implements TaskOperator { +public abstract class BaseTaskExecuteRunnableTimeoutOperator implements TaskExecuteRunnableOperator { - @Autowired private TaskInstanceDao taskInstanceDao; - @Autowired - private MasterRpcClient masterRpcClient; + public BaseTaskExecuteRunnableTimeoutOperator(TaskInstanceDao taskInstanceDao) { + this.taskInstanceDao = taskInstanceDao; + } @Override - public void handle(DefaultTaskExecuteRunnable taskExecuteRunnable) { + public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { // Right now, if the task is running in worker, the timeout strategy will be handled at worker side. // if the task is in master, the timeout strategy will be handled at master side. // todo: we should unify this, the master only need to handle the timeout strategy. and send request to worker @@ -70,17 +60,12 @@ public class TaskTimeoutOperator implements TaskOperator { } + protected abstract void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance); + private void timeoutTaskInstanceInDB(TaskInstance taskInstance) { taskInstance.setState(TaskExecutionStatus.FAILURE); taskInstance.setEndTime(new Date()); taskInstanceDao.updateById(taskInstance); } - private void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) throws RemotingException { - if (StringUtils.isEmpty(taskInstance.getHost())) { - return; - } - TaskKillRequest killCommand = new TaskKillRequest(taskInstance.getId()); - masterRpcClient.send(Host.of(taskInstance.getHost()), killCommand.convert2Command()); - } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableDispatchOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableDispatchOperator.java new file mode 100644 index 0000000000..d5b0802372 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableDispatchOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.server.master.runner.GlobalTaskDispatchWaitingQueue; + +import org.springframework.stereotype.Component; + +@Component +public class LogicTaskExecuteRunnableDispatchOperator extends BaseTaskExecuteRunnableDispatchOperator { + + public LogicTaskExecuteRunnableDispatchOperator(GlobalTaskDispatchWaitingQueue globalTaskDispatchWaitingQueue) { + super(globalTaskDispatchWaitingQueue); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableKillOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableKillOperator.java new file mode 100644 index 0000000000..29ef03906d --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableKillOperator.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse; + +import org.apache.commons.lang3.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class LogicTaskExecuteRunnableKillOperator extends BaseTaskExecuteRunnableKillOperator { + + public LogicTaskExecuteRunnableKillOperator(TaskInstanceDao taskInstanceDao) { + super(taskInstanceDao); + } + + @Override + protected void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) { + if (StringUtils.isEmpty(taskInstance.getHost())) { + log.info("The LogicTaskInstance: {}'s host is null, no need to killRemoteTaskInstance", + taskInstance.getName()); + return; + } + final ILogicTaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ILogicTaskInstanceOperator.class); + final LogicTaskKillRequest logicTaskKillRequest = new LogicTaskKillRequest(taskInstance.getId()); + final LogicTaskKillResponse logicTaskKillResponse = taskInstanceOperator.killLogicTask(logicTaskKillRequest); + log.info("Kill LogicTaskInstance {} on host {} with response {}", taskInstance.getName(), + taskInstance.getHost(), logicTaskKillResponse); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnablePauseOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnablePauseOperator.java new file mode 100644 index 0000000000..af865f05f6 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnablePauseOperator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse; + +import org.apache.commons.lang3.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class LogicTaskExecuteRunnablePauseOperator extends BaseTaskExecuteRunnablePauseOperator { + + @Override + protected void pauseRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) { + if (StringUtils.isEmpty(taskInstance.getHost())) { + log.info("The LogicTaskInstance: {}'s host is null, no need to pauseRemoteTaskInstance", + taskInstance.getName()); + return; + } + final ILogicTaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ILogicTaskInstanceOperator.class); + final LogicTaskPauseRequest logicTaskPauseRequest = new LogicTaskPauseRequest(taskInstance.getId()); + final LogicTaskPauseResponse logicTaskPauseResponse = + taskInstanceOperator.pauseLogicTask(logicTaskPauseRequest); + log.info("Pause LogicTaskInstance: {} on host: {} with response: {}", taskInstance.getName(), + taskInstance.getHost(), logicTaskPauseResponse); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableTimeoutOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableTimeoutOperator.java new file mode 100644 index 0000000000..4cf16bb2c7 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/LogicTaskExecuteRunnableTimeoutOperator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest; +import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse; + +import org.apache.commons.lang3.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class LogicTaskExecuteRunnableTimeoutOperator extends BaseTaskExecuteRunnableTimeoutOperator { + + public LogicTaskExecuteRunnableTimeoutOperator(TaskInstanceDao taskInstanceDao) { + super(taskInstanceDao); + } + + @Override + protected void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) { + if (StringUtils.isEmpty(taskInstance.getHost())) { + log.info("The LogicTaskInstance: {}'s host is null, no need to killRemoteTaskInstance", + taskInstance.getId()); + return; + } + + final ILogicTaskInstanceOperator iLogicTaskInstanceOperator = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ILogicTaskInstanceOperator.class); + + final LogicTaskKillRequest taskInstanceKillRequest = new LogicTaskKillRequest(taskInstance.getId()); + final LogicTaskKillResponse taskInstanceKillResponse = + iLogicTaskInstanceOperator.killLogicTask(taskInstanceKillRequest); + log.info("Timeout kill LogicTaskInstance {} on host {} with response {}", taskInstance.getName(), + taskInstance.getHost(), taskInstanceKillResponse); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskDispatchOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableDispatchOperator.java similarity index 89% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskDispatchOperator.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableDispatchOperator.java index 1280b772e7..5a9f070641 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskDispatchOperator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableDispatchOperator.java @@ -24,13 +24,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class TaskDispatchOperator implements TaskOperator { +public class TaskExecuteRunnableDispatchOperator implements TaskExecuteRunnableOperator { @Autowired private GlobalTaskDispatchWaitingQueue globalTaskDispatchWaitingQueue; @Override - public void handle(DefaultTaskExecuteRunnable taskExecuteRunnable) { + public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { globalTaskDispatchWaitingQueue.submitNeedToDispatchTaskExecuteRunnable(taskExecuteRunnable); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableKillOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableKillOperator.java new file mode 100644 index 0000000000..b80cb79561 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableKillOperator.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse; + +import org.apache.commons.lang3.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskExecuteRunnableKillOperator extends BaseTaskExecuteRunnableKillOperator { + + public TaskExecuteRunnableKillOperator(TaskInstanceDao taskInstanceDao) { + super(taskInstanceDao); + } + + protected void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) { + if (StringUtils.isEmpty(taskInstance.getHost())) { + log.info("TaskInstance {} host is empty, no need to killRemoteTask", taskInstance.getName()); + return; + } + ITaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ITaskInstanceOperator.class); + TaskInstanceKillRequest taskInstanceKillRequest = new TaskInstanceKillRequest(taskInstance.getId()); + TaskInstanceKillResponse taskInstanceKillResponse = taskInstanceOperator.killTask(taskInstanceKillRequest); + log.info("Kill TaskInstance {} on host {} with response {}", taskInstance.getName(), taskInstance.getHost(), + taskInstanceKillResponse); + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperator.java similarity index 89% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskOperator.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperator.java index 6667b364e1..d270103da1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskOperator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperator.java @@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.server.master.runner.operator; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; -public interface TaskOperator { +public interface TaskExecuteRunnableOperator { - void handle(DefaultTaskExecuteRunnable taskExecuteRunnable); + void operate(DefaultTaskExecuteRunnable taskExecuteRunnable); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java new file mode 100644 index 0000000000..b217aff866 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableOperatorManager.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; +import org.apache.dolphinscheduler.server.master.utils.TaskUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TaskExecuteRunnableOperatorManager { + + @Autowired + private TaskExecuteRunnableKillOperator taskKillOperator; + + @Autowired + private LogicTaskExecuteRunnableKillOperator logicTaskKillOperator; + + @Autowired + private TaskExecuteRunnablePauseOperator taskPauseOperator; + + @Autowired + private LogicTaskExecuteRunnablePauseOperator logicTaskPauseOperator; + + @Autowired + private TaskExecuteRunnableDispatchOperator taskDispatchOperator; + + @Autowired + private LogicTaskExecuteRunnableDispatchOperator logicTaskDispatchOperator; + + @Autowired + private TaskExecuteRunnableTimeoutOperator taskTimeoutOperator; + + @Autowired + private LogicTaskExecuteRunnableTimeoutOperator logicTaskTimeoutOperator; + + public TaskExecuteRunnableOperator getTaskKillOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { + if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + return logicTaskKillOperator; + } + return taskKillOperator; + } + + public TaskExecuteRunnableOperator getTaskPauseOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { + if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + return logicTaskPauseOperator; + } + return taskPauseOperator; + } + + public TaskExecuteRunnableOperator getTaskDispatchOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { + if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + return logicTaskDispatchOperator; + } + return taskDispatchOperator; + } + + public TaskExecuteRunnableOperator getTaskTimeoutOperator(DefaultTaskExecuteRunnable defaultTaskExecuteRunnable) { + if (TaskUtils.isMasterTask(defaultTaskExecuteRunnable.getTaskInstance().getTaskType())) { + return logicTaskTimeoutOperator; + } + return taskTimeoutOperator; + } + +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskPauseOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnablePauseOperator.java similarity index 52% rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskPauseOperator.java rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnablePauseOperator.java index 656e402adf..a0636b8504 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskPauseOperator.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnablePauseOperator.java @@ -18,41 +18,42 @@ package org.apache.dolphinscheduler.server.master.runner.operator; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.remote.command.task.TaskPauseRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnable; import org.apache.commons.lang3.StringUtils; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Slf4j @Component -public class TaskPauseOperator implements TaskOperator { - - @Autowired - private MasterRpcClient masterRpcClient; +public class TaskExecuteRunnablePauseOperator implements TaskExecuteRunnableOperator { @Override - public void handle(DefaultTaskExecuteRunnable taskExecuteRunnable) { + public void operate(DefaultTaskExecuteRunnable taskExecuteRunnable) { try { pauseRemoteTaskInstanceInThreadPool(taskExecuteRunnable.getTaskInstance()); } catch (Exception e) { - log.error("Pause MasterTaskExecuteRunnable failed", e); + log.error("Pause DefaultTaskExecuteRunnable failed", e); } } - private void pauseRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) throws RemotingException { + private void pauseRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) { if (StringUtils.isEmpty(taskInstance.getHost())) { - log.info("The task instance: {}'s host is null", taskInstance.getName()); + log.info("The TaskInstance: {} host is null, no need to pauseRemoteTaskInstance", taskInstance.getName()); return; } - masterRpcClient.send(Host.of(taskInstance.getHost()), - new TaskPauseRequest(taskInstance.getId()).convert2Command()); + final ITaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ITaskInstanceOperator.class); + final TaskInstancePauseRequest taskInstancePauseRequest = new TaskInstancePauseRequest(taskInstance.getId()); + final TaskInstancePauseResponse taskInstancePauseResponse = + taskInstanceOperator.pauseTask(taskInstancePauseRequest); + log.info("Pause TaskInstance: {} on host: {} with response: {}", taskInstance.getName(), taskInstance.getHost(), + taskInstancePauseResponse); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableTimeoutOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableTimeoutOperator.java new file mode 100644 index 0000000000..7154f5fa62 --- /dev/null +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/operator/TaskExecuteRunnableTimeoutOperator.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.runner.operator; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse; + +import org.apache.commons.lang3.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskExecuteRunnableTimeoutOperator extends BaseTaskExecuteRunnableTimeoutOperator { + + public TaskExecuteRunnableTimeoutOperator(TaskInstanceDao taskInstanceDao) { + super(taskInstanceDao); + } + + @Override + protected void killRemoteTaskInstanceInThreadPool(TaskInstance taskInstance) { + if (StringUtils.isEmpty(taskInstance.getHost())) { + log.info("TaskInstance {} host is empty, no need to killRemoteTask", taskInstance.getName()); + return; + } + + final ITaskInstanceOperator iTaskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), ITaskInstanceOperator.class); + + final TaskInstanceKillRequest taskInstanceKillRequest = new TaskInstanceKillRequest(taskInstance.getId()); + final TaskInstanceKillResponse taskInstanceKillResponse = + iTaskInstanceOperator.killTask(taskInstanceKillRequest); + log.info("Timeout kill TaskInstance {} in host {} with response {}", taskInstance.getName(), + taskInstance.getHost(), taskInstanceKillResponse); + } +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java index 453446fda2..25b53c8d6d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java @@ -30,16 +30,15 @@ import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.DynamicInputParameter; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction; import org.apache.dolphinscheduler.server.master.runner.task.BaseAsyncLogicTask; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -78,8 +77,6 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { private TaskInstance taskInstance; - private final MasterRpcClient masterRpcClient; - private boolean haveBeenCanceled = false; public DynamicLogicTask(TaskExecutionContext taskExecutionContext, @@ -87,7 +84,6 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { TaskInstanceDao taskInstanceDao, SubWorkflowService subWorkflowService, ProcessService processService, - MasterRpcClient masterRpcClient, ProcessDefinitionMapper processDefineMapper, CommandMapper commandMapper) { super(taskExecutionContext, @@ -96,7 +92,6 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { this.processInstanceDao = processInstanceDao; this.subWorkflowService = subWorkflowService; this.processService = processService; - this.masterRpcClient = masterRpcClient; this.processDefineMapper = processDefineMapper; this.commandMapper = commandMapper; @@ -280,7 +275,7 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { sendToSubProcess(taskExecutionContext, subProcessInstance); log.info("Success send [{}] request to SubWorkflow's master: {}", stopStatus, subProcessInstance.getHost()); - } catch (RemotingException e) { + } catch (Exception e) { throw new MasterTaskExecuteException( String.format("Send stop request to SubWorkflow's master: %s failed", subProcessInstance.getHost()), @@ -289,16 +284,17 @@ public class DynamicLogicTask extends BaseAsyncLogicTask { } } - private void sendToSubProcess(TaskExecutionContext taskExecutionContext, - ProcessInstance subProcessInstance) throws RemotingException { - WorkflowStateEventChangeRequest stateEventChangeCommand = new WorkflowStateEventChangeRequest( + private void sendToSubProcess(TaskExecutionContext taskExecutionContext, ProcessInstance subProcessInstance) { + final ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(subProcessInstance.getHost(), ITaskInstanceExecutionEventListener.class); + final WorkflowInstanceStateChangeEvent workflowInstanceStateChangeEvent = new WorkflowInstanceStateChangeEvent( taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId(), subProcessInstance.getState(), subProcessInstance.getId(), 0); - Host host = new Host(subProcessInstance.getHost()); - masterRpcClient.send(host, stateEventChangeCommand.convert2Command()); + iTaskInstanceExecutionEventListener.onWorkflowInstanceInstanceStateChange(workflowInstanceStateChangeEvent); } public boolean isCancel() { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java index cbcc50ada6..caca32d431 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.subworkflow.SubWorkflowService; @@ -54,14 +53,11 @@ public class DynamicLogicTaskPluginFactory implements ILogicTaskPluginFactory() { })); this.processInstanceExecCacheManager = processInstanceExecCacheManager; this.processInstanceDao = processInstanceDao; - this.masterRpcClient = masterRpcClient; } @Override @@ -95,7 +91,7 @@ public class SubWorkflowLogicTask extends BaseAsyncLogicTask processInstanceCacheMap = new HashMap<>(); for (TaskInstance taskInstance : needFailoverTaskInstanceList) { - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(taskInstance.getProcessInstanceId(), - taskInstance.getId())) { - try { - ProcessInstance processInstance = processInstanceCacheMap.computeIfAbsent( - taskInstance.getProcessInstanceId(), k -> { - WorkflowExecuteRunnable workflowExecuteRunnable = cacheManager.getByProcessInstanceId( - taskInstance.getProcessInstanceId()); - if (workflowExecuteRunnable == null) { - return null; - } - return workflowExecuteRunnable.getWorkflowExecuteContext() - .getWorkflowInstance(); - }); - if (!checkTaskInstanceNeedFailover(needFailoverWorkerStartTime, processInstance, taskInstance)) { - log.info("Worker[{}] the current taskInstance doesn't need to failover", workerHost); - continue; - } - log.info( - "Worker[{}] failover: begin to failover taskInstance, will set the status to NEED_FAULT_TOLERANCE", - workerHost); - failoverTaskInstance(processInstance, taskInstance); - log.info("Worker[{}] failover: Finish failover taskInstance", workerHost); - } catch (Exception ex) { - log.info("Worker[{}] failover taskInstance occur exception", workerHost, ex); + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(taskInstance.getProcessInstanceId(), taskInstance.getId()); + ProcessInstance processInstance = processInstanceCacheMap.computeIfAbsent( + taskInstance.getProcessInstanceId(), k -> { + WorkflowExecuteRunnable workflowExecuteRunnable = cacheManager.getByProcessInstanceId( + taskInstance.getProcessInstanceId()); + if (workflowExecuteRunnable == null) { + return null; + } + return workflowExecuteRunnable.getWorkflowExecuteContext() + .getWorkflowInstance(); + }); + if (!checkTaskInstanceNeedFailover(needFailoverWorkerStartTime, processInstance, taskInstance)) { + log.info("Worker[{}] the current taskInstance doesn't need to failover", workerHost); + continue; } + log.info( + "Worker[{}] failover: begin to failover taskInstance, will set the status to NEED_FAULT_TOLERANCE", + workerHost); + failoverTaskInstance(processInstance, taskInstance); + log.info("Worker[{}] failover: Finish failover taskInstance", workerHost); + } catch (Exception ex) { + log.info("Worker[{}] failover taskInstance occur exception", workerHost, ex); + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } failoverTimeCost.stop(); @@ -170,19 +168,7 @@ public class WorkerFailoverService { taskInstance.setProcessInstance(processInstance); if (!TaskUtils.isMasterTask(taskInstance.getTaskType())) { - log.info("The failover taskInstance is not master task"); - TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() - .buildWorkflowInstanceHost(masterConfig.getMasterAddress()) - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .buildProcessDefinitionRelatedInfo(processInstance.getProcessDefinition()) - .create(); - - if (masterConfig.isKillApplicationWhenTaskFailover()) { - // only kill yarn/k8s job if exists , the local thread has exited - log.info("TaskInstance failover begin kill the task related yarn or k8s job"); - ProcessUtils.killApplication(logClient, taskExecutionContext); - } + killYarnTask(taskInstance, processInstance); } else { log.info("The failover taskInstance is a master task, no need to failover in worker failover"); } @@ -271,4 +257,30 @@ public class WorkerFailoverService { } return Optional.ofNullable(serverStartupTime); } + + private void killYarnTask(TaskInstance taskInstance, ProcessInstance processInstance) { + try { + if (!masterConfig.isKillApplicationWhenTaskFailover()) { + return; + } + if (StringUtils.isEmpty(taskInstance.getHost()) || StringUtils.isEmpty(taskInstance.getLogPath())) { + return; + } + TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get() + .buildWorkflowInstanceHost(masterConfig.getMasterAddress()) + .buildTaskInstanceRelatedInfo(taskInstance) + .buildProcessInstanceRelatedInfo(processInstance) + .buildProcessDefinitionRelatedInfo(processInstance.getProcessDefinition()) + .create(); + // only kill yarn/k8s job if exists , the local thread has exited + log.info("TaskInstance failover begin kill the task related yarn or k8s job"); + IWorkerLogService iWorkerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IWorkerLogService.class); + GetAppIdResponse getAppIdResponse = + iWorkerLogService.getAppId(new GetAppIdRequest(taskInstance.getId(), taskInstance.getLogPath())); + ProcessUtils.killApplication(getAppIdResponse.getAppIds(), taskExecutionContext); + } catch (Exception ex) { + log.error("Kill yarn task error", ex); + } + } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java deleted file mode 100644 index 9588c7f914..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.dispatch; - -import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchRequest; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; -import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; - -import org.mockito.Mockito; - -/** - * for test use only - */ -public class ExecutionContextTestUtils { - - public static ExecutionContext getExecutionContext(int port) { - TaskInstance taskInstance = Mockito.mock(TaskInstance.class); - ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class); - processDefinition.setId(0); - ProcessInstance processInstance = new ProcessInstance(); - processInstance.setId(0); - processInstance.setCommandType(CommandType.COMPLEMENT_DATA); - taskInstance.setProcessInstance(processInstance); - TaskExecutionContext context = TaskExecutionContextBuilder.get() - .buildWorkflowInstanceHost("127.0.0.1:5678") - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .buildProcessDefinitionRelatedInfo(processDefinition) - .create(); - - TaskDispatchRequest requestCommand = new TaskDispatchRequest(context); - Message message = requestCommand.convert2Command(); - - ExecutionContext executionContext = new ExecutionContext(message, ExecutorType.WORKER, taskInstance); - executionContext.setHost(Host.of(NetUtils.getAddr(port))); - - return executionContext; - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java deleted file mode 100644 index de971d487e..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.dispatch.executor; - -import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchRequest; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.builder.TaskExecutionContextBuilder; -import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; -import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; -import org.apache.dolphinscheduler.server.worker.processor.WorkerTaskDispatchProcessor; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -/** - * netty executor manager test - */ -@ExtendWith(SpringExtension.class) -@Disabled -public class NettyExecutorManagerTest { - - @Autowired - private NettyExecutorManager nettyExecutorManager; - @Test - public void testExecute() throws ExecuteException { - final NettyServerConfig serverConfig = new NettyServerConfig(); - serverConfig.setListenPort(30000); - NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(new WorkerTaskDispatchProcessor()); - nettyRemotingServer.start(); - TaskInstance taskInstance = Mockito.mock(TaskInstance.class); - ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class); - ProcessInstance processInstance = new ProcessInstance(); - processInstance.setCommandType(CommandType.COMPLEMENT_DATA); - taskInstance.setProcessInstance(processInstance); - TaskExecutionContext context = TaskExecutionContextBuilder.get() - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .buildProcessDefinitionRelatedInfo(processDefinition) - .create(); - ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance); - executionContext.setHost(Host.of(NetUtils.getAddr(serverConfig.getListenPort()))); - Assertions.assertDoesNotThrow(() -> nettyExecutorManager.execute(executionContext)); - nettyRemotingServer.close(); - } - - @Test - public void testExecuteWithException() { - TaskInstance taskInstance = Mockito.mock(TaskInstance.class); - ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class); - ProcessInstance processInstance = new ProcessInstance(); - processInstance.setCommandType(CommandType.COMPLEMENT_DATA); - taskInstance.setProcessInstance(processInstance); - TaskExecutionContext context = TaskExecutionContextBuilder.get() - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .buildProcessDefinitionRelatedInfo(processDefinition) - .create(); - ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance); - executionContext.setHost(Host.of(NetUtils.getAddr(4444))); - Assertions.assertThrows(ExecuteException.class, () -> { - nettyExecutorManager.execute(executionContext); - }); - - } - private Message toCommand(TaskExecutionContext taskExecutionContext) { - TaskDispatchRequest requestCommand = new TaskDispatchRequest(taskExecutionContext); - return requestCommand.convert2Command(); - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java deleted file mode 100644 index 042fda90f4..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.dispatch.host; - -import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.master.dispatch.ExecutionContextTestUtils; -import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; -import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; - -import java.util.Optional; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.google.common.collect.Sets; - -/** - * round robin host manager test - */ -@ExtendWith(MockitoExtension.class) -public class RoundRobinHostManagerTest { - - @Mock - private ServerNodeManager serverNodeManager; - - @InjectMocks - RoundRobinHostManager roundRobinHostManager; - - @Test - public void testSelectWithEmptyResult() throws WorkerGroupNotFoundException { - Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(null); - ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); - Optional emptyHost = roundRobinHostManager.select(context.getWorkerGroup()); - Assertions.assertFalse(emptyHost.isPresent()); - } - - @Test - public void testSelectWithResult() throws WorkerGroupNotFoundException { - Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22")); - Mockito.when(serverNodeManager.getWorkerNodeInfo("192.168.1.1:22")) - .thenReturn(Optional.of(new WorkerHeartBeat())); - ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); - Optional host = roundRobinHostManager.select(context.getWorkerGroup()); - Assertions.assertTrue(host.isPresent()); - Assertions.assertTrue(host.get().getAddress().equalsIgnoreCase("192.168.1.1:22")); - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java deleted file mode 100644 index cf44b66ed7..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessorTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.common.enums.CacheType; -import org.apache.dolphinscheduler.dao.entity.Tenant; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.cache.CacheExpireRequest; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.cache.Cache; -import org.springframework.cache.CacheManager; - -import io.netty.channel.Channel; - -/** - * task ack processor test - */ -@ExtendWith(MockitoExtension.class) -public class CacheProcessorTest { - - @InjectMocks - private CacheProcessor cacheProcessor = new CacheProcessor(); - - @Mock - private Channel channel; - - @Mock - private CacheManager cacheManager; - - @Mock - private Cache cache; - - @BeforeEach - public void before() { - Mockito.when(cacheManager.getCache(CacheType.TENANT.getCacheName())).thenReturn(cache); - } - - @Test - public void testProcess() { - Tenant tenant = new Tenant(); - tenant.setId(1); - CacheExpireRequest cacheExpireRequest = new CacheExpireRequest(CacheType.TENANT, "1"); - Message message = cacheExpireRequest.convert2Command(); - - cacheProcessor.process(channel, message); - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java deleted file mode 100644 index f5eaa4424d..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessage; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; -import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; -import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.service.process.ProcessService; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.netty.channel.Channel; - -/** - * task ack processor test - */ -@ExtendWith(MockitoExtension.class) -public class TaskAckProcessorTest { - - private TaskExecuteRunningProcessor taskExecuteRunningProcessor; - private TaskEventService taskEventService; - private ProcessService processService; - private TaskExecuteRunningMessage taskExecuteRunningMessage; - private TaskEvent taskResponseEvent; - private Channel channel; - - private MockedStatic mockedStaticSpringApplicationContext; - - @BeforeEach - public void before() { - mockedStaticSpringApplicationContext = Mockito.mockStatic(SpringApplicationContext.class); - - taskEventService = Mockito.mock(TaskEventService.class); - mockedStaticSpringApplicationContext.when(() -> SpringApplicationContext.getBean(TaskEventService.class)) - .thenReturn(taskEventService); - - processService = Mockito.mock(ProcessService.class); - mockedStaticSpringApplicationContext.when(() -> SpringApplicationContext.getBean(ProcessService.class)) - .thenReturn(processService); - - taskExecuteRunningProcessor = new TaskExecuteRunningProcessor(); - - channel = Mockito.mock(Channel.class); - taskResponseEvent = Mockito.mock(TaskEvent.class); - - taskExecuteRunningMessage = new TaskExecuteRunningMessage("127.0.0.1:5678", - " 127.0.0.1:1234", - System.currentTimeMillis()); - taskExecuteRunningMessage.setStatus(TaskExecutionStatus.RUNNING_EXECUTION); - taskExecuteRunningMessage.setExecutePath("/dolphinscheduler/worker"); - taskExecuteRunningMessage.setHost("localhost"); - taskExecuteRunningMessage.setLogPath("/temp/worker.log"); - taskExecuteRunningMessage.setStartTime(System.currentTimeMillis()); - taskExecuteRunningMessage.setTaskInstanceId(1); - taskExecuteRunningMessage.setProcessInstanceId(1); - } - - @AfterEach - public void after() { - mockedStaticSpringApplicationContext.close(); - } - - @Test - public void testProcess() { - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java deleted file mode 100644 index b3280885a1..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessorTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor; - -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskKillResponse; - -import java.util.ArrayList; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import io.netty.channel.Channel; - -/** - * task response processor test - */ -public class TaskKillResponseProcessorTest { - - private TaskKillResponseProcessor taskKillResponseProcessor; - - private TaskKillResponse taskKillResponse; - - private Channel channel; - - @BeforeEach - public void before() { - taskKillResponseProcessor = new TaskKillResponseProcessor(); - channel = Mockito.mock(Channel.class); - taskKillResponse = new TaskKillResponse(); - taskKillResponse.setAppIds( - new ArrayList() { - - { - add("task_1"); - } - }); - taskKillResponse.setHost("localhost"); - taskKillResponse.setProcessId(1); - taskKillResponse.setStatus(TaskExecutionStatus.RUNNING_EXECUTION); - taskKillResponse.setTaskInstanceId(1); - - } - - @Test - public void testProcess() { - Message message = taskKillResponse.convert2Command(1); - Assertions.assertEquals(MessageType.RESPONSE, message.getType()); - taskKillResponseProcessor.process(channel, message); - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java deleted file mode 100644 index 92031ee824..0000000000 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.master.processor.queue; - -import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessage; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessage; -import org.apache.dolphinscheduler.server.master.cache.impl.ProcessInstanceExecCacheManagerImpl; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; -import org.apache.dolphinscheduler.server.master.utils.DataQualityResultOperator; -import org.apache.dolphinscheduler.service.process.ProcessService; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.netty.channel.Channel; - -@ExtendWith(MockitoExtension.class) -public class TaskResponseServiceTest { - - @Mock(name = "processService") - private ProcessService processService; - - @InjectMocks - TaskEventService taskEventService; - - @Mock - private Channel channel; - - private TaskEvent ackEvent; - - private TaskEvent resultEvent; - - private TaskInstance taskInstance; - - @Mock - private ProcessInstanceExecCacheManagerImpl processInstanceExecCacheManager; - - @Mock - private DataQualityResultOperator dataQualityResultOperator; - - @Mock - private WorkflowExecuteThreadPool workflowExecuteThreadPool; - - @Mock - private TaskExecuteThreadPool taskExecuteThreadPool; - - @BeforeEach - public void before() { - taskEventService.start(); - - TaskExecuteRunningMessage taskExecuteRunningMessage = new TaskExecuteRunningMessage("127.0.0.1:5678", - "127.0.0.1:1234", - System.currentTimeMillis()); - taskExecuteRunningMessage.setProcessId(1); - taskExecuteRunningMessage.setTaskInstanceId(22); - taskExecuteRunningMessage.setStatus(TaskExecutionStatus.RUNNING_EXECUTION); - taskExecuteRunningMessage.setExecutePath("path"); - taskExecuteRunningMessage.setLogPath("logPath"); - taskExecuteRunningMessage.setHost("127.*.*.*"); - taskExecuteRunningMessage.setStartTime(System.currentTimeMillis()); - - ackEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, - channel, - taskExecuteRunningMessage.getMessageSenderAddress()); - - TaskExecuteResultMessage taskExecuteResultMessage = new TaskExecuteResultMessage(NetUtils.getAddr(1234), - NetUtils.getAddr(5678), - System.currentTimeMillis()); - taskExecuteResultMessage.setProcessInstanceId(1); - taskExecuteResultMessage.setTaskInstanceId(22); - taskExecuteResultMessage.setStatus(TaskExecutionStatus.SUCCESS.getCode()); - taskExecuteResultMessage.setEndTime(System.currentTimeMillis()); - taskExecuteResultMessage.setVarPool("varPol"); - taskExecuteResultMessage.setAppIds("ids"); - taskExecuteResultMessage.setProcessId(1); - resultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, - channel, - taskExecuteResultMessage.getMessageSenderAddress()); - - taskInstance = new TaskInstance(); - taskInstance.setId(22); - taskInstance.setState(TaskExecutionStatus.RUNNING_EXECUTION); - } - - @Test - public void testAddResponse() { - taskEventService.addEvent(ackEvent); - taskEventService.addEvent(resultEvent); - } - - @AfterEach - public void after() { - if (taskEventService != null) { - taskEventService.stop(); - } - } -} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java index 3aff6d65e8..da96be2066 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java @@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory; import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; @@ -127,7 +126,6 @@ public class WorkflowExecuteRunnableTest { stateWheelExecuteThread = Mockito.mock(StateWheelExecuteThread.class); curingGlobalParamsService = Mockito.mock(CuringParamsService.class); - MasterRpcClient masterRpcClient = Mockito.mock(MasterRpcClient.class); ProcessAlertManager processAlertManager = Mockito.mock(ProcessAlertManager.class); WorkflowExecuteContext workflowExecuteContext = Mockito.mock(WorkflowExecuteContext.class); Mockito.when(workflowExecuteContext.getWorkflowInstance()).thenReturn(processInstance); @@ -141,7 +139,6 @@ public class WorkflowExecuteRunnableTest { commandService, processService, processInstanceDao, - masterRpcClient, processAlertManager, config, stateWheelExecuteThread, diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcherTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcherTest.java index a4f169c63b..c9710209e5 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcherTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcherTest.java @@ -17,10 +17,9 @@ package org.apache.dolphinscheduler.server.master.runner.dispatcher; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable; import org.junit.jupiter.api.Assertions; @@ -37,11 +36,10 @@ public class MasterTaskDispatcherTest { TaskEventService taskEventService = Mockito.mock(TaskEventService.class); MasterConfig masterConfig = Mockito.mock(MasterConfig.class); Mockito.when(masterConfig.getMasterAddress()).thenReturn("localhost:5678"); - MasterRpcClient masterRpcClient = Mockito.mock(MasterRpcClient.class); TaskExecuteRunnable taskExecuteRunnable = Mockito.mock(TaskExecuteRunnable.class); MasterTaskDispatcher masterTaskDispatcher = - new MasterTaskDispatcher(taskEventService, masterConfig, masterRpcClient); + new MasterTaskDispatcher(taskEventService, masterConfig); Host taskInstanceDispatchHost = masterTaskDispatcher.getTaskInstanceDispatchHost(taskExecuteRunnable) .orElseThrow(() -> new IllegalArgumentException("Cannot get the ")); Assertions.assertEquals(masterConfig.getMasterAddress(), taskInstanceDispatchHost.getAddress()); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcherTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcherTest.java index b1b5e308ab..9b447aada3 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcherTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcherTest.java @@ -17,13 +17,12 @@ package org.apache.dolphinscheduler.server.master.runner.dispatcher; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.WorkerGroupNotFoundException; import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.server.master.runner.execute.TaskExecuteRunnable; import java.util.Optional; @@ -41,11 +40,10 @@ public class WorkerTaskDispatcherTest { public void getTaskInstanceDispatchHost() throws WorkerGroupNotFoundException { TaskEventService taskEventService = Mockito.mock(TaskEventService.class); MasterConfig masterConfig = Mockito.mock(MasterConfig.class); - MasterRpcClient masterRpcClient = Mockito.mock(MasterRpcClient.class); HostManager hostManager = Mockito.mock(HostManager.class); Mockito.when(hostManager.select(Mockito.any())).thenReturn(Optional.of(Host.of("localhost:1234"))); WorkerTaskDispatcher workerTaskDispatcher = - new WorkerTaskDispatcher(taskEventService, masterConfig, masterRpcClient, hostManager); + new WorkerTaskDispatcher(taskEventService, masterConfig, hostManager); TaskExecuteRunnable taskExecuteRunnable = Mockito.mock(TaskExecuteRunnable.class); Mockito.when(taskExecuteRunnable.getTaskExecutionContext()).thenReturn(new TaskExecutionContext()); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityTaskExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityTaskExecuteRunnableTest.java index 979a467148..3a3a80b3ca 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityTaskExecuteRunnableTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/execute/PriorityTaskExecuteRunnableTest.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.server.master.runner.operator.TaskOperatorManager; +import org.apache.dolphinscheduler.server.master.runner.operator.TaskExecuteRunnableOperatorManager; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,7 +30,7 @@ public class PriorityTaskExecuteRunnableTest { @Test public void testCompareTo() { - TaskOperatorManager taskOperatorManager = new TaskOperatorManager(); + TaskExecuteRunnableOperatorManager taskOperatorManager = new TaskExecuteRunnableOperatorManager(); ProcessInstance workflowInstance = new ProcessInstance(); workflowInstance.setId(1); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java index 15fccaf9a1..a5b9f5a371 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskTest.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.DynamicInputParameter; import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters; -import org.apache.dolphinscheduler.server.master.rpc.MasterRpcClient; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.subworkflow.SubWorkflowService; @@ -61,9 +60,6 @@ class DynamicLogicTaskTest { @Mock private ProcessService processService; - @Mock - private MasterRpcClient masterRpcClient; - @Mock private ProcessDefinitionMapper processDefineMapper; @@ -94,7 +90,6 @@ class DynamicLogicTaskTest { taskInstanceDao, subWorkflowService, processService, - masterRpcClient, processDefineMapper, commandMapper); } @@ -126,7 +121,6 @@ class DynamicLogicTaskTest { taskInstanceDao, subWorkflowService, processService, - masterRpcClient, processDefineMapper, commandMapper); diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java index b84b227712..2081d2dd7a 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java @@ -40,7 +40,6 @@ import org.apache.dolphinscheduler.server.master.runner.IWorkflowExecuteContext; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.apache.dolphinscheduler.service.log.LogClient; import org.apache.dolphinscheduler.service.process.ProcessService; import java.util.ArrayList; @@ -87,9 +86,6 @@ public class FailoverServiceTest { @Mock private ProcessInstanceExecCacheManager processInstanceExecCacheManager; - @Mock - private LogClient logClient; - private static int masterPort = 5678; private static int workerPort = 1234; @@ -116,7 +112,6 @@ public class FailoverServiceTest { processService, workflowExecuteThreadPool, cacheManager, - logClient, taskInstanceDao); failoverService = new FailoverService(masterFailoverService, workerFailoverService); diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml index 59f71c50d5..aaf7065323 100644 --- a/dolphinscheduler-microbench/pom.xml +++ b/dolphinscheduler-microbench/pom.xml @@ -70,10 +70,6 @@ org.slf4j slf4j-api - - org.apache.dolphinscheduler - dolphinscheduler-remote - diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java deleted file mode 100644 index 1f3f3213d1..0000000000 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.microbench.common; - -import org.apache.dolphinscheduler.microbench.base.AbstractBaseBenchmark; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.rpc.client.IRpcClient; -import org.apache.dolphinscheduler.rpc.client.RpcClient; -import org.apache.dolphinscheduler.rpc.remote.NettyClient; -import org.apache.dolphinscheduler.rpc.remote.NettyServer; - -import java.util.concurrent.TimeUnit; - -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; - -@Warmup(iterations = 5, time = 1) -@Measurement(iterations = 10, time = 1) -@State(Scope.Benchmark) -@BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) -public class RpcTest extends AbstractBaseBenchmark { - - private NettyServer nettyServer; - - private IUserService userService; - - private Host host; - private IRpcClient rpcClient = new RpcClient(); - - @Setup - public void before() throws Exception { - nettyServer = new NettyServer(new NettyServerConfig()); - IRpcClient rpcClient = new RpcClient(); - host = new Host("127.0.0.1", 12346); - userService = rpcClient.create(IUserService.class, host); - - } - - @Benchmark - @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void sendTest() throws Exception { - - userService = rpcClient.create(IUserService.class, host); - Integer result = userService.hi(1); - } - - @TearDown - public void after() { - NettyClient.getInstance().close(); - nettyServer.close(); - } - -} diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml deleted file mode 100644 index edc8dd50be..0000000000 --- a/dolphinscheduler-remote/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - 4.0.0 - - org.apache.dolphinscheduler - dolphinscheduler - dev-SNAPSHOT - - - dolphinscheduler-remote - - dolphinscheduler-remote - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - - - - - org.apache.dolphinscheduler - dolphinscheduler-common - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - - - io.netty - netty-all - - - org.slf4j - slf4j-api - - - net.bytebuddy - byte-buddy - - - - - io.protostuff - protostuff-core - - - - io.protostuff - protostuff-runtime - - - - org.reflections - reflections - - - - com.google.guava - guava - - - org.springframework - spring-context - - - - diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java deleted file mode 100644 index f7bb944b5e..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.codec; - -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageContext; -import org.apache.dolphinscheduler.remote.command.MessageHeader; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ReplayingDecoder; - -/** - * netty decoder - */ -@Slf4j -public class NettyDecoder extends ReplayingDecoder { - - public NettyDecoder() { - super(State.MAGIC); - } - - private final MessageHeader messageHeader = new MessageHeader(); - - /** - * decode - * - * @param ctx channel handler context - * @param in byte buffer - * @param out out content - */ - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { - switch (state()) { - case MAGIC: - checkMagic(in.readByte()); - checkpoint(State.VERSION); - // fallthru - case VERSION: - checkVersion(in.readByte()); - checkpoint(State.COMMAND); - // fallthru - case COMMAND: - messageHeader.setType(in.readByte()); - checkpoint(State.OPAQUE); - // fallthru - case OPAQUE: - messageHeader.setOpaque(in.readLong()); - checkpoint(State.CONTEXT_LENGTH); - // fallthru - case CONTEXT_LENGTH: - messageHeader.setContextLength(in.readInt()); - checkpoint(State.CONTEXT); - // fallthru - case CONTEXT: - byte[] context = new byte[messageHeader.getContextLength()]; - in.readBytes(context); - messageHeader.setContext(context); - checkpoint(State.BODY_LENGTH); - // fallthru - case BODY_LENGTH: - messageHeader.setBodyLength(in.readInt()); - checkpoint(State.BODY); - // fallthru - case BODY: - byte[] body = new byte[messageHeader.getBodyLength()]; - in.readBytes(body); - // - Message packet = new Message(); - packet.setType(commandType(messageHeader.getType())); - packet.setOpaque(messageHeader.getOpaque()); - packet.setContext(MessageContext.valueOf(messageHeader.getContext())); - packet.setBody(body); - out.add(packet); - // - checkpoint(State.MAGIC); - break; - default: - log.warn("unknown decoder state {}", state()); - } - } - - /** - * get command type - * - * @param type type - */ - private MessageType commandType(byte type) { - for (MessageType ct : MessageType.values()) { - if (ct.ordinal() == type) { - return ct; - } - } - return null; - } - - /** - * check magic - * - * @param magic magic - */ - private void checkMagic(byte magic) { - if (magic != Message.MAGIC) { - throw new IllegalArgumentException("illegal packet [magic]" + magic); - } - } - - /** - * check version - */ - private void checkVersion(byte version) { - if (version != Message.VERSION) { - throw new IllegalArgumentException("illegal protocol [version]" + version); - } - } - - enum State { - MAGIC, - VERSION, - COMMAND, - OPAQUE, - CONTEXT_LENGTH, - CONTEXT, - BODY_LENGTH, - BODY; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseMessage.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseMessage.java deleted file mode 100644 index a5d73c26b6..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseMessage.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -import java.io.Serializable; - -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * This is the base class for rpc message. - *

- * Since we use async mode, the client send a message and will wait the target server - * send ack for the message, the client will retry during a while if he doesn't receive an ack. - *

- * When there is a network error, the server cannot send ack to the client by the origin channel, - * since the client has closed the channel, so the server need to know the command source. - */ -@Data -@NoArgsConstructor -public abstract class BaseMessage implements Serializable, RequestMessageBuilder { - - private static final long serialVersionUID = -1L; - - /** - * If the message receiver want to send ack to the sender, need to use this address. - */ - protected String messageSenderAddress; - - /** - * The message receiver address. - */ - protected String messageReceiverAddress; - - protected long messageSendTime; - - protected BaseMessage(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) { - this.messageSenderAddress = messageSenderAddress; - this.messageReceiverAddress = messageReceiverAddress; - this.messageSendTime = messageSendTime; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Message.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Message.java deleted file mode 100644 index 767cc483fd..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Message.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -import java.io.Serializable; -import java.util.concurrent.atomic.AtomicLong; - -/** - * receive task log request command and content fill - * for netty data serializable transfer - */ -public class Message implements Serializable { - - private static final long serialVersionUID = -1L; - - private static final AtomicLong REQUEST_ID = new AtomicLong(1); - - public static final byte MAGIC = (byte) 0xbabe; - public static final byte VERSION = 0; - - public Message() { - this.opaque = REQUEST_ID.getAndIncrement(); - } - - public Message(long opaque) { - this.opaque = opaque; - } - - /** - * command type - */ - private MessageType type; - - /** - * request unique identification - */ - private long opaque; - - /** - * request context - */ - private MessageContext context = new MessageContext(); - - /** - * data body - */ - private byte[] body; - - public MessageType getType() { - return type; - } - - public void setType(MessageType type) { - this.type = type; - } - - public long getOpaque() { - return opaque; - } - - public void setOpaque(long opaque) { - this.opaque = opaque; - } - - public byte[] getBody() { - return body; - } - - public void setBody(byte[] body) { - this.body = body; - } - - public MessageContext getContext() { - return context; - } - - public void setContext(MessageContext context) { - this.context = context; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (int) (opaque ^ (opaque >>> 32)); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Message other = (Message) obj; - return opaque == other.opaque; - } - - @Override - public String toString() { - return "Command [type=" + type + ", opaque=" + opaque + ", bodyLen=" + (body == null ? 0 : body.length) + "]"; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageContext.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageContext.java deleted file mode 100644 index f53a3771a3..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageContext.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; - -import java.io.Serializable; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * command context - */ -public class MessageContext implements Serializable { - - private Map items = new LinkedHashMap<>(); - - public Map getItems() { - return items; - } - - public void setItems(Map items) { - this.items = items; - } - - public void put(String key, String value) { - items.put(key, value); - } - - public String get(String key) { - return items.get(key); - } - - public byte[] toBytes() { - return JSONUtils.toJsonByteArray(this); - } - - public static MessageContext valueOf(byte[] src) { - return JSONUtils.parseObject(src, MessageContext.class); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageHeader.java deleted file mode 100644 index 271b5dec62..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageHeader.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -import java.io.Serializable; - -import lombok.Data; - -/** - * command header - */ -@Data -public class MessageHeader implements Serializable { - - /** - * type - */ - private byte type; - - /** - * request unique identification - */ - private long opaque; - - /** - * context length - */ - private int contextLength; - - /** - * context - */ - private byte[] context; - - /** - * body length - */ - private int bodyLength; - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageType.java deleted file mode 100644 index 407e92296b..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/MessageType.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -public enum MessageType { - - RESPONSE, - - GET_APP_ID_REQUEST, - - REMOVE_TAK_LOG_REQUEST, - - ROLL_VIEW_LOG_REQUEST, - - VIEW_WHOLE_LOG_REQUEST, - - GET_LOG_BYTES_REQUEST, - - /** - * task execute start, from api to master - */ - TASK_EXECUTE_START, - - TASK_DISPATCH_REQUEST, - TASK_DISPATCH_RESPONSE, - - /** - * task execute running, from worker to master - */ - TASK_EXECUTE_RUNNING_MESSAGE, - - /** - * task execute running ack, from master to worker - */ - TASK_EXECUTE_RUNNING_MESSAGE_ACK, - - /** - * task execute response, from worker to master - */ - TASK_EXECUTE_RESULT_MESSAGE, - - /** - * task execute response ack, from master to worker - */ - TASK_EXECUTE_RESULT_MESSAGE_ACK, - - TASK_KILL_REQUEST, - - /** - * task savepoint, for stream task - */ - TASK_SAVEPOINT_REQUEST, - - HEART_BEAT, - - PING, - - PONG, - - ALERT_SEND_REQUEST, - - WORKFLOW_HOST_CHANGE_REQUEST, - - /** - * state event request - */ - STATE_EVENT_REQUEST, - /** - * cache expire - */ - CACHE_EXPIRE, - /** - * task state event request - */ - TASK_FORCE_STATE_EVENT_REQUEST, - /** - * task state event request - */ - TASK_WAKEUP_EVENT_REQUEST, - - /** - * workflow executing data request, from api to master - */ - WORKFLOW_EXECUTING_DATA_REQUEST, - - /** - * update taskInstance's PID request - */ - TASK_UPDATE_RUNTIME_MESSAGE, - - /** - * update taskInstance's PID response ack, from master to worker - */ - TASK_UPDATE_RUNTIME_MESSAGE_ACK, - - WORKFLOW_METRICS_CLEANUP, - - PAUSE_TASK_INSTANCE, - ; - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/RequestMessageBuilder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/RequestMessageBuilder.java deleted file mode 100644 index 32933804a4..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/RequestMessageBuilder.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -import org.apache.dolphinscheduler.remote.utils.JsonSerializer; - -import java.io.Serializable; - -public interface RequestMessageBuilder extends Serializable { - - default Message convert2Command() { - Message message = new Message(); - message.setType(getCommandType()); - byte[] body = JsonSerializer.serialize(this); - message.setBody(body); - return message; - } - - default Message convert2Command(int opaque) { - Message message = new Message(opaque); - message.setType(getCommandType()); - byte[] body = JsonSerializer.serialize(this); - message.setBody(body); - return message; - } - - MessageType getCommandType(); -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ResponseMessageBuilder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ResponseMessageBuilder.java deleted file mode 100644 index 3b208a4fcd..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/ResponseMessageBuilder.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command; - -import org.apache.dolphinscheduler.remote.utils.JsonSerializer; - -import java.io.Serializable; - -public interface ResponseMessageBuilder extends Serializable { - - default Message convert2Command(long opaque) { - Message message = new Message(opaque); - message.setType(getCommandType()); - byte[] body = JsonSerializer.serialize(this); - message.setBody(body); - return message; - } - - MessageType getCommandType(); -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponse.java deleted file mode 100644 index b051a8e9f6..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponse.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * get log bytes response command - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class GetLogBytesResponse implements ResponseMessageBuilder { - - /** - * log byte data - */ - private byte[] data; - - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequest.java deleted file mode 100644 index 4c958614ca..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * remove task log request command - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class RemoveTaskLogRequest implements RequestMessageBuilder { - - /** - * log path - */ - private String path; - - @Override - public MessageType getCommandType() { - return MessageType.REMOVE_TAK_LOG_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponse.java deleted file mode 100644 index 6884a06fad..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * remove task log request command - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class RemoveTaskLogResponse implements ResponseMessageBuilder { - - /* - * TaskPriorityQueueConsumer.* log path - */ - private Boolean status; - - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponse.java deleted file mode 100644 index 30cc8bc895..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * roll view log response command - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class RollViewLogResponse implements ResponseMessageBuilder { - - /** - * response data - */ - private String msg; - - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessage.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessage.java deleted file mode 100644 index 5a326a501e..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessage.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.ToString; - -/** - * execute task response command - */ -@Data -@NoArgsConstructor -@ToString(callSuper = true) -@EqualsAndHashCode(callSuper = true) -public class TaskExecuteResultMessage extends BaseMessage { - - public TaskExecuteResultMessage(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) { - super(messageSenderAddress, messageReceiverAddress, messageSendTime); - } - - /** - * task instance id - */ - private int taskInstanceId; - - /** - * process instance id - */ - private int processInstanceId; - - /** - * status - */ - private int status; - - /** - * startTime - */ - private long startTime; - - /** - * host - */ - private String host; - - /** - * logPath - */ - private String logPath; - - /** - * executePath - */ - private String executePath; - - /** - * end time - */ - private long endTime; - - /** - * processId - */ - private int processId; - - /** - * appIds - */ - private String appIds; - - /** - * varPool string - */ - private String varPool; - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RESULT_MESSAGE; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessageAck.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessageAck.java deleted file mode 100644 index 109eb1498e..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteResultMessageAck.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.ToString; - -/** - * task execute response ack command - * from master to worker - */ -@Data -@NoArgsConstructor -@ToString(callSuper = true) -@EqualsAndHashCode(callSuper = true) -public class TaskExecuteResultMessageAck extends BaseMessage { - - private int taskInstanceId; - private boolean success; - - public TaskExecuteResultMessageAck(boolean success, - int taskInstanceId, - String sourceServerAddress, - String messageReceiverAddress, - long messageSendTime) { - super(sourceServerAddress, messageReceiverAddress, messageSendTime); - this.success = success; - this.taskInstanceId = taskInstanceId; - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RESULT_MESSAGE_ACK; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessage.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessage.java deleted file mode 100644 index 9556686bcc..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessage.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.ToString; - -/** - * Task running message, means the task is running in worker. - */ -@Data -@NoArgsConstructor -@ToString(callSuper = true) -@EqualsAndHashCode(callSuper = true) -public class TaskExecuteRunningMessage extends BaseMessage { - - /** - * taskInstanceId - */ - private int taskInstanceId; - - /** - * process instance id - */ - private int processInstanceId; - - /** - * startTime - */ - private long startTime; - - /** - * host - */ - private String host; - - /** - * status - */ - private TaskExecutionStatus status; - - /** - * logPath - */ - private String logPath; - - /** - * executePath - */ - private String executePath; - - /** - * processId - */ - private int processId; - - /** - * appIds - */ - private String appIds; - - public TaskExecuteRunningMessage(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) { - super(messageSenderAddress, messageReceiverAddress, messageSendTime); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RUNNING_MESSAGE; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessageAck.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessageAck.java deleted file mode 100644 index 21733add5b..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskExecuteRunningMessageAck.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.ToString; - -/** - * task execute running ack command - * from master to worker - */ -@Data -@NoArgsConstructor -@ToString(callSuper = true) -@EqualsAndHashCode(callSuper = true) -public class TaskExecuteRunningMessageAck extends BaseMessage { - - private boolean success; - private int taskInstanceId; - - public TaskExecuteRunningMessageAck(boolean success, - int taskInstanceId, - String messageSenderAddress, - String messageReceiverAddress, - long messageSendTime) { - super(messageSenderAddress, messageReceiverAddress, messageSendTime); - this.success = success; - this.taskInstanceId = taskInstanceId; - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RUNNING_MESSAGE_ACK; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeMessage.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeMessage.java deleted file mode 100644 index e90727d46b..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskUpdateRuntimeMessage.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import lombok.ToString; - -/** - * Task running message, means the task is running in worker. - */ -@Data -@NoArgsConstructor -@ToString(callSuper = true) -@EqualsAndHashCode(callSuper = true) -public class TaskUpdateRuntimeMessage extends BaseMessage { - - /** - * taskInstanceId - */ - private int taskInstanceId; - - /** - * process instance id - */ - private int processInstanceId; - - /** - * startTime - */ - private long startTime; - - /** - * host - */ - private String host; - - /** - * logPath - */ - private String logPath; - - /** - * processId - */ - private int processId; - - public TaskUpdateRuntimeMessage(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) { - super(messageSenderAddress, messageReceiverAddress, messageSendTime); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_UPDATE_RUNTIME_MESSAGE; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeResponse.java deleted file mode 100644 index 44b78d186f..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/WorkflowHostChangeResponse.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.task; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class WorkflowHostChangeResponse implements ResponseMessageBuilder { - - boolean success; - - public static WorkflowHostChangeResponse success() { - WorkflowHostChangeResponse response = new WorkflowHostChangeResponse(); - response.setSuccess(true); - return response; - } - - public static WorkflowHostChangeResponse failed() { - WorkflowHostChangeResponse response = new WorkflowHostChangeResponse(); - response.setSuccess(false); - return response; - } - - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataRequest.java deleted file mode 100644 index 9a6285bb0a..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.workflow; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * workflow executing data request, from api to master - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class WorkflowExecutingDataRequest implements RequestMessageBuilder { - - private Integer processInstanceId; - - @Override - public MessageType getCommandType() { - return MessageType.WORKFLOW_EXECUTING_DATA_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataResponse.java deleted file mode 100644 index ef0f4084b2..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowExecutingDataResponse.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.workflow; - -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder; -import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * workflow executing data response, from master to api - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class WorkflowExecutingDataResponse implements ResponseMessageBuilder { - - private WorkflowExecuteDto workflowExecuteDto; - - @Override - public MessageType getCommandType() { - return MessageType.RESPONSE; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingServerFactory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingServerFactory.java deleted file mode 100644 index 9669795b77..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingServerFactory.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.factory; - -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class NettyRemotingServerFactory { - - public NettyRemotingServer buildNettyRemotingServer(int listenPort) { - NettyServerConfig serverConfig = new NettyServerConfig(); - serverConfig.setListenPort(listenPort); - return new NettyRemotingServer(serverConfig); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java deleted file mode 100644 index 7dfa37f447..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.handler; - -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.future.ResponseFuture; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.ChannelUtils; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.Pair; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; - -import lombok.extern.slf4j.Slf4j; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleStateEvent; - -/** - * netty client request handler - */ -@ChannelHandler.Sharable -@Slf4j -public class NettyClientHandler extends ChannelInboundHandlerAdapter { - - /** - * netty client - */ - private final NettyRemotingClient nettyRemotingClient; - - private static byte[] heartBeatData = "heart_beat".getBytes(); - - /** - * callback thread executor - */ - private final ExecutorService callbackExecutor; - - /** - * processors - */ - private final ConcurrentHashMap> processors; - - /** - * default executor - */ - private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS); - - public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) { - this.nettyRemotingClient = nettyRemotingClient; - this.callbackExecutor = callbackExecutor; - this.processors = new ConcurrentHashMap<>(); - } - - /** - * When the current channel is not active, - * the current channel has reached the end of its life cycle - * - * @param ctx channel handler context - */ - @Override - public void channelInactive(ChannelHandlerContext ctx) { - nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); - ctx.channel().close(); - } - - /** - * The current channel reads data from the remote - * - * @param ctx channel handler context - * @param msg message - */ - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { - processReceived(ctx.channel(), (Message) msg); - } - - /** - * register processor - * - * @param messageType command type - * @param processor processor - */ - public void registerProcessor(final MessageType messageType, final NettyRequestProcessor processor) { - this.registerProcessor(messageType, processor, null); - } - - /** - * register processor - * - * @param messageType command type - * @param processor processor - * @param executor thread executor - */ - public void registerProcessor(final MessageType messageType, final NettyRequestProcessor processor, - final ExecutorService executor) { - ExecutorService executorRef = executor; - if (executorRef == null) { - executorRef = defaultExecutor; - } - this.processors.putIfAbsent(messageType, new Pair<>(processor, executorRef)); - } - - /** - * process received logic - * - * @param message command - */ - private void processReceived(final Channel channel, final Message message) { - ResponseFuture future = ResponseFuture.getFuture(message.getOpaque()); - if (future != null) { - future.setResponseCommand(message); - future.release(); - if (future.getInvokeCallback() != null) { - future.removeFuture(); - this.callbackExecutor.submit(future::executeInvokeCallback); - } else { - future.putResponse(message); - } - } else { - processByCommandType(channel, message); - } - } - - public void processByCommandType(final Channel channel, final Message message) { - final Pair pair = processors.get(message.getType()); - if (pair != null) { - Runnable run = () -> { - try { - pair.getLeft().process(channel, message); - } catch (Exception e) { - log.error(String.format("process command %s exception", message), e); - } - }; - try { - pair.getRight().submit(run); - } catch (RejectedExecutionException e) { - log.warn("thread pool is full, discard command {} from {}", message, - ChannelUtils.getRemoteAddress(channel)); - } - } else { - log.warn("receive response {}, but not matched any request ", message); - } - } - - /** - * caught exception - * - * @param ctx channel handler context - * @param cause cause - */ - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - log.error("exceptionCaught : {}", cause.getMessage(), cause); - nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel())); - ctx.channel().close(); - } - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof IdleStateEvent) { - Message heartBeat = new Message(); - heartBeat.setType(MessageType.HEART_BEAT); - heartBeat.setBody(heartBeatData); - ctx.channel().writeAndFlush(heartBeat) - .addListener(ChannelFutureListener.CLOSE_ON_FAILURE); - if (log.isDebugEnabled()) { - log.debug("Client send heart beat to: {}", ChannelUtils.getRemoteAddress(ctx.channel())); - } - } else { - super.userEventTriggered(ctx, evt); - } - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java deleted file mode 100644 index 788c593ba3..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.handler; - -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.ChannelUtils; -import org.apache.dolphinscheduler.remote.utils.Pair; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; - -import lombok.extern.slf4j.Slf4j; -import io.netty.channel.Channel; -import io.netty.channel.ChannelConfig; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleStateEvent; - -/** - * netty server request handler - */ -@ChannelHandler.Sharable -@Slf4j -public class NettyServerHandler extends ChannelInboundHandlerAdapter { - - /** - * netty remote server - */ - private final NettyRemotingServer nettyRemotingServer; - - /** - * server processors queue - */ - private final ConcurrentHashMap> processors = - new ConcurrentHashMap<>(); - - public NettyServerHandler(NettyRemotingServer nettyRemotingServer) { - this.nettyRemotingServer = nettyRemotingServer; - } - - /** - * When the current channel is not active, - * the current channel has reached the end of its life cycle - * - * @param ctx channel handler context - */ - @Override - public void channelInactive(ChannelHandlerContext ctx) { - ctx.channel().close(); - } - - /** - * The current channel reads data from the remote end - * - * @param ctx channel handler context - * @param msg message - */ - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { - processReceived(ctx.channel(), (Message) msg); - } - - /** - * register processor - * - * @param messageType command type - * @param processor processor - */ - public void registerProcessor(final MessageType messageType, final NettyRequestProcessor processor) { - this.registerProcessor(messageType, processor, null); - } - - /** - * register processor - * - * @param messageType command type - * @param processor processor - * @param executor thread executor - */ - public void registerProcessor(final MessageType messageType, final NettyRequestProcessor processor, - final ExecutorService executor) { - ExecutorService executorRef = executor; - if (executorRef == null) { - executorRef = nettyRemotingServer.getDefaultExecutor(); - } - this.processors.putIfAbsent(messageType, new Pair<>(processor, executorRef)); - } - - /** - * process received logic - * - * @param channel channel - * @param msg message - */ - private void processReceived(final Channel channel, final Message msg) { - final MessageType messageType = msg.getType(); - if (MessageType.HEART_BEAT.equals(messageType)) { - if (log.isDebugEnabled()) { - log.debug("server receive heart beat from: host: {}", ChannelUtils.getRemoteAddress(channel)); - } - return; - } - final Pair pair = processors.get(messageType); - if (pair != null) { - Runnable r = () -> { - try { - pair.getLeft().process(channel, msg); - } catch (Exception ex) { - log.error("process msg {} error", msg, ex); - } - }; - try { - pair.getRight().submit(r); - } catch (RejectedExecutionException e) { - log.warn("thread pool is full, discard msg {} from {}", msg, ChannelUtils.getRemoteAddress(channel)); - } - } else { - log.warn("commandType {} not support", messageType); - } - } - - /** - * caught exception - * - * @param ctx channel handler context - * @param cause cause - */ - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - log.error("exceptionCaught : {}", cause.getMessage(), cause); - ctx.channel().close(); - } - - /** - * channel write changed - * - * @param ctx channel handler context - */ - @Override - public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { - Channel ch = ctx.channel(); - ChannelConfig config = ch.config(); - - if (!ch.isWritable()) { - if (log.isWarnEnabled()) { - log.warn("{} is not writable, over high water level : {}", - ch, config.getWriteBufferHighWaterMark()); - } - - config.setAutoRead(false); - } else { - if (log.isWarnEnabled()) { - log.warn("{} is writable, to low water : {}", - ch, config.getWriteBufferLowWaterMark()); - } - config.setAutoRead(true); - } - } - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof IdleStateEvent) { - ctx.channel().close(); - } else { - super.userEventTriggered(ctx, evt); - } - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetAppIdProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetAppIdProcessor.java deleted file mode 100644 index e5081dc191..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetAppIdProcessor.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import static org.apache.dolphinscheduler.common.constants.Constants.APPID_COLLECT; -import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_COLLECT_WAY; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.log.GetAppIdRequest; -import org.apache.dolphinscheduler.remote.command.log.GetAppIdResponse; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Component -@Slf4j -public class GetAppIdProcessor extends BaseLogProcessor implements NettyRequestProcessor { - - @Override - public void process(Channel channel, Message message) { - GetAppIdRequest getAppIdRequest = - JSONUtils.parseObject(message.getBody(), GetAppIdRequest.class); - TaskExecutionContext taskExecutionContext = - TaskExecutionContextCacheManager.getByTaskInstanceId(getAppIdRequest.getTaskInstanceId()); - String appInfoPath = taskExecutionContext.getAppInfoPath(); - String logPath = getAppIdRequest.getLogPath(); - List appIds = LogUtils.getAppIds(logPath, appInfoPath, - PropertyUtils.getString(APPID_COLLECT, DEFAULT_COLLECT_WAY)); - channel.writeAndFlush( - new GetAppIdResponse(appIds).convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.GET_APP_ID_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetLogBytesProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetLogBytesProcessor.java deleted file mode 100644 index 38fd39dc68..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/GetLogBytesProcessor.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.log.GetLogBytesRequest; -import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponse; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Component -@Slf4j -public class GetLogBytesProcessor extends BaseLogProcessor implements NettyRequestProcessor { - - @Override - public void process(Channel channel, Message message) { - GetLogBytesRequest getLogRequest = JSONUtils.parseObject( - message.getBody(), GetLogBytesRequest.class); - String path = getLogRequest.getPath(); - byte[] bytes = LogUtils.getFileContentBytes(path); - GetLogBytesResponse getLogResponse = new GetLogBytesResponse(bytes); - channel.writeAndFlush(getLogResponse.convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.GET_LOG_BYTES_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java deleted file mode 100644 index e4f7173f64..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.utils.ChannelUtils; -import org.apache.dolphinscheduler.remote.utils.Host; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; - -/** - * callback channel - */ -public class NettyRemoteChannel { - - /** - * channel - */ - private final Channel channel; - - /** - * request unique identification - */ - private final long opaque; - - /** - * master host - */ - private final Host host; - - public NettyRemoteChannel(Channel channel, long opaque) { - this.channel = channel; - this.host = ChannelUtils.toAddress(channel); - this.opaque = opaque; - } - - public NettyRemoteChannel(Channel channel) { - this.channel = channel; - this.host = ChannelUtils.toAddress(channel); - this.opaque = -1; - } - - public Channel getChannel() { - return channel; - } - - public long getOpaque() { - return opaque; - } - - public Host getHost() { - return host; - } - - public boolean isActive() { - return this.channel.isActive(); - } - - public ChannelFuture writeAndFlush(Message message) { - return this.channel.writeAndFlush(message); - } - - public void close() { - this.channel.close(); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java deleted file mode 100644 index d731eddd60..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.remote.processor; - -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import io.netty.channel.Channel; - -/** - * netty request processor - */ -public interface NettyRequestProcessor { - - /** - * process logic - * @param channel channel - * @param message command - */ - void process(final Channel channel, final Message message); - - MessageType getCommandType(); -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RemoveTaskLogProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RemoveTaskLogProcessor.java deleted file mode 100644 index e52366ada1..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RemoveTaskLogProcessor.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequest; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponse; - -import java.io.File; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Component -@Slf4j -public class RemoveTaskLogProcessor extends BaseLogProcessor implements NettyRequestProcessor { - - @Override - public void process(Channel channel, Message message) { - RemoveTaskLogRequest removeTaskLogRequest = JSONUtils.parseObject( - message.getBody(), RemoveTaskLogRequest.class); - - String taskLogPath = removeTaskLogRequest.getPath(); - File taskLogFile = new File(taskLogPath); - boolean status = true; - try { - if (taskLogFile.exists()) { - status = taskLogFile.delete(); - } - } catch (Exception e) { - status = false; - } - - RemoveTaskLogResponse removeTaskLogResponse = new RemoveTaskLogResponse(status); - channel.writeAndFlush(removeTaskLogResponse.convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.REMOVE_TAK_LOG_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RollViewLogRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RollViewLogRequest.java deleted file mode 100644 index bf92f60be3..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/RollViewLogRequest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponse; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Component -@Slf4j -public class RollViewLogRequest extends BaseLogProcessor implements NettyRequestProcessor { - - @Override - public void process(Channel channel, Message message) { - org.apache.dolphinscheduler.remote.command.log.RollViewLogRequest rollViewLogRequest = JSONUtils.parseObject( - message.getBody(), org.apache.dolphinscheduler.remote.command.log.RollViewLogRequest.class); - - String rollViewLogPath = rollViewLogRequest.getPath(); - - List lines = LogUtils.readPartFileContent(rollViewLogPath, - rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit()); - - String logContent = LogUtils.rollViewLogLines(lines); - RollViewLogResponse rollViewLogRequestResponse = - new RollViewLogResponse(logContent); - channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.ROLL_VIEW_LOG_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java deleted file mode 100644 index bd2fbcaf2c..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import static org.apache.dolphinscheduler.common.constants.Constants.HTTP_CONNECTION_REQUEST_TIMEOUT; -import static org.apache.dolphinscheduler.common.constants.Constants.SLEEP_TIME_MILLIS; - -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; - -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Service; - -import io.netty.channel.Channel; - -/** - * task callback service - */ -@Service -@Slf4j -public class StateEventCallbackService { - - private static final int[] RETRY_BACKOFF = {1, 2, 3, 5, 10, 20, 40, 100, 100, 100, 100, 200, 200, 200}; - - /** - * remote channels - */ - private static final ConcurrentHashMap REMOTE_CHANNELS = new ConcurrentHashMap<>(); - - /** - * netty remoting client - */ - private final NettyRemotingClient nettyRemotingClient; - - public StateEventCallbackService() { - final NettyClientConfig clientConfig = new NettyClientConfig(); - this.nettyRemotingClient = new NettyRemotingClient(clientConfig); - } - - /** - * add callback channel - * - * @param channel channel - */ - public void addRemoteChannel(String host, NettyRemoteChannel channel) { - REMOTE_CHANNELS.put(host, channel); - } - - /** - * get callback channel - * - * @param host - * @return callback channel - */ - private Optional newRemoteChannel(Host host) { - Channel newChannel; - NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(host.getAddress()); - if (nettyRemoteChannel != null) { - if (nettyRemoteChannel.isActive()) { - return Optional.of(nettyRemoteChannel); - } - } - newChannel = nettyRemotingClient.getChannel(host); - if (newChannel != null) { - return Optional.of(newRemoteChannel(newChannel, host.getAddress())); - } - return Optional.empty(); - } - - public long pause(int ntries) { - return SLEEP_TIME_MILLIS * RETRY_BACKOFF[ntries % RETRY_BACKOFF.length]; - } - - private NettyRemoteChannel newRemoteChannel(Channel newChannel, long opaque, String host) { - NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel, opaque); - addRemoteChannel(host, remoteChannel); - return remoteChannel; - } - - private NettyRemoteChannel newRemoteChannel(Channel newChannel, String host) { - NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel); - addRemoteChannel(host, remoteChannel); - return remoteChannel; - } - - /** - * remove callback channels - */ - public void remove(String host) { - REMOTE_CHANNELS.remove(host); - } - - /** - * Send the command to target host, this method doesn't guarantee the command send success. - * - * @param host target host - * @param message command need to send - */ - public void sendResult(Host host, Message message) { - log.info("send result, host:{}, command:{}", host.getAddress(), message.toString()); - newRemoteChannel(host).ifPresent(nettyRemoteChannel -> { - nettyRemoteChannel.writeAndFlush(message); - }); - } - - /** - * send sync and return response command - * @param host - * @param requestMessage - * @return - * @throws RemotingException - * @throws InterruptedException - */ - public Message sendSync(Host host, Message requestMessage) { - try { - return this.nettyRemotingClient.sendSync(host, requestMessage, HTTP_CONNECTION_REQUEST_TIMEOUT); - } catch (InterruptedException e) { - log.error("send sync fail, host:{}, command:{}", host, requestMessage, e); - Thread.currentThread().interrupt(); - } catch (RemotingException e) { - log.error("send sync fail, host:{}, command:{}", host, requestMessage, e); - } finally { - this.nettyRemotingClient.closeChannel(host); - } - return null; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/ViewWholeLogProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/ViewWholeLogProcessor.java deleted file mode 100644 index 5bbbebd671..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/ViewWholeLogProcessor.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.log.ViewLogRequest; -import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseResponse; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Component -@Slf4j -public class ViewWholeLogProcessor extends BaseLogProcessor implements NettyRequestProcessor { - - @Override - public void process(Channel channel, Message message) { - ViewLogRequest viewLogRequest = JSONUtils.parseObject( - message.getBody(), ViewLogRequest.class); - String viewLogPath = viewLogRequest.getPath(); - String msg = LogUtils.readWholeFileContent(viewLogPath); - ViewLogResponseResponse viewLogResponse = new ViewLogResponseResponse(msg); - channel.writeAndFlush(viewLogResponse.convert2Command(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.VIEW_WHOLE_LOG_REQUEST; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java deleted file mode 100644 index 9857c9f4a3..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.base; - -import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Rpc - */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface Rpc { - - /** - * number of retries - */ - int retries() default 3; - - boolean async() default false; - - boolean ack() default false; - - boolean callBack() default false; - - // todo It is better to set the timeout period for synchronous calls - - /** - * When it is asynchronous transmission, callback must be set - */ - Class serviceCallback() default AbstractRpcCallBack.class; - - Class ackCallback() default AbstractRpcCallBack.class; - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java deleted file mode 100644 index 331b0fdf59..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.client; - -import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; -import org.apache.dolphinscheduler.rpc.common.ConsumerConfigConstants; - -/** - * We will cache the consumer configuration, when the rpc call is generated, the consumer configuration will be first obtained from here - */ -public class ConsumerConfig { - - private Class serviceCallBackClass; - - private Class ackCallBackClass; - - private String serviceName; - - private Boolean async = ConsumerConfigConstants.DEFAULT_SYNC; - - private Boolean callBack = ConsumerConfigConstants.DEFAULT_CALL_BACK; - - private Integer retries = ConsumerConfigConstants.DEFAULT_RETRIES; - - public Class getServiceCallBackClass() { - return serviceCallBackClass; - } - - void setServiceCallBackClass(Class serviceCallBackClass) { - this.serviceCallBackClass = serviceCallBackClass; - } - - public Class getAckCallBackClass() { - return ackCallBackClass; - } - - void setAckCallBackClass(Class ackCallBackClass) { - this.ackCallBackClass = ackCallBackClass; - } - - public String getServiceName() { - return serviceName; - } - - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - public Boolean getAsync() { - return async; - } - - void setAsync(Boolean async) { - this.async = async; - } - - Integer getRetries() { - return retries; - } - - void setRetries(Integer retries) { - this.retries = retries; - } - - public Boolean getCallBack() { - return callBack; - } - - public void setCallBack(Boolean callBack) { - this.callBack = callBack; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java deleted file mode 100644 index 4c8c9f4a07..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.client; - -import java.util.concurrent.ConcurrentHashMap; - -/** - * ConsumerConfigCache - */ -public class ConsumerConfigCache { - - private ConsumerConfigCache() { - throw new IllegalStateException("Utility class"); - } - - private static ConcurrentHashMap consumerMap = new ConcurrentHashMap<>(); - - public static ConsumerConfig getConfigByServersName(String serviceName) { - return consumerMap.get(serviceName); - } - - static void putConfig(String serviceName, ConsumerConfig consumerConfig) { - consumerMap.putIfAbsent(serviceName, consumerConfig); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java deleted file mode 100644 index 4456d54996..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.client; - -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.rpc.base.Rpc; -import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; -import org.apache.dolphinscheduler.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.rpc.protocol.EventType; -import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; -import org.apache.dolphinscheduler.rpc.remote.NettyClient; -import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; - -import java.lang.reflect.Method; - -import net.bytebuddy.implementation.bind.annotation.AllArguments; -import net.bytebuddy.implementation.bind.annotation.Origin; -import net.bytebuddy.implementation.bind.annotation.RuntimeType; - -/** - * ConsumerInterceptor - */ -public class ConsumerInterceptor { - - private Host host; - - private NettyClient nettyClient = NettyClient.getInstance(); - - ConsumerInterceptor(Host host) { - this.host = host; - } - - @RuntimeType - public Object intercept(@AllArguments Object[] args, @Origin Method method) throws RemotingException { - RpcRequest request = buildReq(args, method); - - String serviceName = method.getDeclaringClass().getSimpleName() + method.getName(); - ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); - if (null == consumerConfig) { - consumerConfig = cacheServiceConfig(method, serviceName); - } - boolean async = consumerConfig.getAsync(); - - int retries = consumerConfig.getRetries(); - - RpcProtocol protocol = buildProtocol(request); - - while (retries-- > 0) { - RpcResponse rsp; - rsp = nettyClient.sendMsg(host, protocol, async); - // success - if (null != rsp && rsp.getStatus() == 0) { - return rsp.getResult(); - } - } - // execute fail - throw new RemotingException("send msg error"); - - } - - private RpcRequest buildReq(Object[] args, Method method) { - RpcRequest request = new RpcRequest(); - request.setClassName(method.getDeclaringClass().getSimpleName()); - request.setMethodName(method.getName()); - request.setParameterTypes(method.getParameterTypes()); - request.setParameters(args); - - return request; - } - - private ConsumerConfig cacheServiceConfig(Method method, String serviceName) { - ConsumerConfig consumerConfig = new ConsumerConfig(); - consumerConfig.setServiceName(serviceName); - boolean annotationPresent = method.isAnnotationPresent(Rpc.class); - if (annotationPresent) { - Rpc rpc = method.getAnnotation(Rpc.class); - consumerConfig.setAsync(rpc.async()); - consumerConfig.setServiceCallBackClass(rpc.serviceCallback()); - if (!rpc.serviceCallback().isInstance(AbstractRpcCallBack.class)) { - consumerConfig.setCallBack(true); - } - consumerConfig.setAckCallBackClass(rpc.ackCallback()); - consumerConfig.setRetries(rpc.retries()); - } - - ConsumerConfigCache.putConfig(serviceName, consumerConfig); - - return consumerConfig; - } - - private RpcProtocol buildProtocol(RpcRequest req) { - RpcProtocol protocol = new RpcProtocol<>(); - MessageHeader header = new MessageHeader(); - header.setRequestId(RpcRequestTable.getRequestId()); - header.setEventType(EventType.REQUEST.getType()); - header.setSerialization(RpcSerializer.PROTOSTUFF.getType()); - protocol.setMsgHeader(header); - protocol.setBody(req); - return protocol; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java deleted file mode 100644 index 02f99b4f62..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.client; - -import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; - -import org.apache.dolphinscheduler.remote.utils.Host; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.implementation.MethodDelegation; - -/** - * RpcClient - */ -public class RpcClient implements IRpcClient { - - @Override - public T create(Class clazz, Host host) throws Exception { - return new ByteBuddy() - .subclass(clazz) - .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(host))) - .make() - .load(getClass().getClassLoader()) - .getLoaded() - .getDeclaredConstructor().newInstance(); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java deleted file mode 100644 index 2c0cf0cf8a..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.client; - -import org.apache.dolphinscheduler.rpc.future.RpcFuture; - -/** - * Rpc Request Cache - */ -public class RpcRequestCache { - - private RpcFuture rpcFuture; - - private String serviceName; - - public RpcFuture getRpcFuture() { - return rpcFuture; - } - - public void setRpcFuture(RpcFuture rpcFuture) { - this.rpcFuture = rpcFuture; - } - - public String getServiceName() { - return serviceName; - } - - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java deleted file mode 100644 index 4d47522698..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.client; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; - -/** - * RpcRequestTable - */ -public class RpcRequestTable { - - private RpcRequestTable() { - throw new IllegalStateException("Utility class"); - } - - private static AtomicLong requestIdGen = new AtomicLong(0); - - private static ConcurrentHashMap requestMap = new ConcurrentHashMap<>(); - - public static void put(long requestId, RpcRequestCache rpcRequestCache) { - requestMap.put(requestId, rpcRequestCache); - } - - public static RpcRequestCache get(Long requestId) { - return requestMap.get(requestId); - } - - public static void remove(Long requestId) { - requestMap.remove(requestId); - } - - public static long getRequestId() { - return requestIdGen.incrementAndGet(); - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java deleted file mode 100644 index 5263615994..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.codec; - -import org.apache.dolphinscheduler.rpc.protocol.EventType; -import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocolConstants; -import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; -import org.apache.dolphinscheduler.rpc.serializer.Serializer; - -import java.util.List; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; - -/** - * NettyDecoder - */ -public class NettyDecoder extends ByteToMessageDecoder { - - private Class genericClass; - - public NettyDecoder(Class genericClass) { - this.genericClass = genericClass; - } - - @Override - protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, - List list) throws Exception { - if (byteBuf.readableBytes() < RpcProtocolConstants.HEADER_LENGTH) { - return; - } - - byteBuf.markReaderIndex(); - - short magic = byteBuf.readShort(); - - if (RpcProtocolConstants.MAGIC != magic) { - throw new IllegalArgumentException("magic number is illegal, " + magic); - } - byte eventType = byteBuf.readByte(); - byte version = byteBuf.readByte(); - byte serialization = byteBuf.readByte(); - long requestId = byteBuf.readLong(); - int dataLength = byteBuf.readInt(); - byte[] data = new byte[dataLength]; - - RpcProtocol rpcProtocol = new RpcProtocol(); - - MessageHeader header = new MessageHeader(); - header.setVersion(version); - header.setSerialization(serialization); - header.setRequestId(requestId); - header.setEventType(eventType); - header.setMsgLength(dataLength); - byteBuf.readBytes(data); - rpcProtocol.setMsgHeader(header); - if (eventType != EventType.HEARTBEAT.getType()) { - Serializer serializer = RpcSerializer.getSerializerByType(serialization); - Object obj = serializer.deserialize(data, genericClass); - rpcProtocol.setBody(obj); - } - list.add(rpcProtocol); - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java deleted file mode 100644 index d076e4760a..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.codec; - -import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; -import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; -import org.apache.dolphinscheduler.rpc.serializer.Serializer; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToByteEncoder; - -/** - * NettyEncoder - */ -public class NettyEncoder extends MessageToByteEncoder> { - - @Override - protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, - ByteBuf byteBuf) throws Exception { - MessageHeader msgHeader = msg.getMsgHeader(); - byteBuf.writeShort(msgHeader.getMagic()); - byteBuf.writeByte(msgHeader.getEventType()); - byteBuf.writeByte(msgHeader.getVersion()); - byteBuf.writeByte(msgHeader.getSerialization()); - byteBuf.writeLong(msgHeader.getRequestId()); - byte[] data = new byte[0]; - int msgLength = msgHeader.getMsgLength(); - Serializer rpcSerializer = RpcSerializer.getSerializerByType(msgHeader.getSerialization()); - if (null != rpcSerializer) { - data = rpcSerializer.serialize(msg.getBody()); - msgLength = data.length; - } - byteBuf.writeInt(msgLength); - byteBuf.writeBytes(data); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java deleted file mode 100644 index e07c55488c..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.common; - -/** - * ConsumerConfigConstants - */ -public class ConsumerConfigConstants { - - private ConsumerConfigConstants() { - throw new IllegalStateException("Utility class"); - } - - public static final Boolean DEFAULT_SYNC = false; - - public static final Integer DEFAULT_RETRIES = 3; - - public static final Boolean DEFAULT_CALL_BACK = false; -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java deleted file mode 100644 index 5810993fc0..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.common; - -public enum RequestEventType { - - HEARTBEAT((byte) 1, "heartbeat"), - BUSINESS((byte) 2, "business request"); - - private Byte type; - - private String description; - - RequestEventType(Byte type, String description) { - this.type = type; - this.description = description; - } - - public Byte getType() { - return type; - } - - public String getDescription() { - return description; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java deleted file mode 100644 index 877d40fd0e..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.common; - -/** - * RpcRequest - */ -public class RpcRequest { - - private String className; - private String methodName; - private Class[] parameterTypes; - private Object[] parameters; - - /** - * @see RequestEventType - */ - private Byte eventType = RequestEventType.BUSINESS.getType(); - - private Boolean ack; - - public Byte getEventType() { - return eventType; - } - - public void setEventType(Byte eventType) { - this.eventType = eventType; - } - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } - - public String getMethodName() { - return methodName; - } - - public void setMethodName(String methodName) { - this.methodName = methodName; - } - - public Class[] getParameterTypes() { - return parameterTypes; - } - - public void setParameterTypes(Class[] parameterTypes) { - this.parameterTypes = parameterTypes; - } - - public Object[] getParameters() { - return parameters; - } - - public void setParameters(Object[] parameters) { - this.parameters = parameters; - } - - public Boolean getAck() { - return ack; - } - - public void setAck(Boolean ack) { - this.ack = ack; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java deleted file mode 100644 index 6aafc039aa..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.common; - -/** - * RpcResponse - */ -public class RpcResponse { - - private String requestId; - private String msg; - private Object result; - private Byte status; - - /** - * @see ResponseEventType - */ - private Byte responseType; - - public String getRequestId() { - return requestId; - } - - public void setRequestId(String requestId) { - this.requestId = requestId; - } - - public String getMsg() { - return msg; - } - - public void setMsg(String msg) { - this.msg = msg; - } - - public Object getResult() { - return result; - } - - public void setResult(Object result) { - this.result = result; - } - - public Byte getStatus() { - return status; - } - - public void setStatus(Byte status) { - this.status = status; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java deleted file mode 100644 index 1a2ce2efee..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.common; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.ThreadPoolExecutor.DiscardPolicy; -import java.util.concurrent.TimeUnit; - -public enum ThreadPoolManager { - - INSTANCE; - - ExecutorService executorService; - - private static final int WORK_QUEUE_SIZE = 200; - private static final long KEEP_ALIVE_TIME = 60; - - ThreadPoolManager() { - executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 4, KEEP_ALIVE_TIME, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(WORK_QUEUE_SIZE), - new DiscardPolicy()); - } - - public void addExecuteTask(Runnable task) { - executorService.submit(task); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java deleted file mode 100644 index e1746d8c84..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.config; - -import org.apache.dolphinscheduler.rpc.base.RpcService; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -import org.reflections.Reflections; - -/** - * ServiceBean find all rpcService - */ -@Slf4j -public class ServiceBean { - - private static Map serviceMap = new HashMap<>(); - - private static AtomicBoolean initialized = new AtomicBoolean(false); - - private ServiceBean() { - throw new IllegalStateException("Utility class"); - } - - private static synchronized void init() { - // todo config - if (initialized.get()) { - return; - } - Reflections f = new Reflections("org.apache.dolphinscheduler."); - List> list = new ArrayList<>(f.getTypesAnnotatedWith(RpcService.class)); - list.forEach(rpcClass -> { - RpcService rpcService = rpcClass.getAnnotation(RpcService.class); - serviceMap.put(rpcService.value(), rpcClass); - log.info("load rpc service {}", rpcService.value()); - }); - initialized.set(true); - } - - public static Class getServiceClass(String className) { - if (initialized.get()) { - return serviceMap.get(className); - } else { - init(); - } - return serviceMap.get(className); - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java deleted file mode 100644 index b782d86b0f..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.future; - -import org.apache.dolphinscheduler.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -/** - * RpcFuture - */ -public class RpcFuture implements Future { - - private CountDownLatch latch = new CountDownLatch(1); - - private RpcResponse response; - - private RpcRequest request; - - private long requestId; - - public RpcFuture(RpcRequest rpcRequest, long requestId) { - this.request = rpcRequest; - this.requestId = requestId; - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return false; - } - - @Override - public RpcResponse get() throws InterruptedException { - // the timeout period should be defined by the business party - boolean success = latch.await(5, TimeUnit.SECONDS); - if (!success) { - throw new RuntimeException("Timeout exception. Request id: " + this.requestId - + ". Request class name: " + this.request.getClassName() - + ". Request method: " + this.request.getMethodName()); - } - return response; - } - - @Override - public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException { - boolean success = latch.await(timeout, unit); - if (!success) { - throw new RuntimeException("Timeout exception. Request id: " + requestId - + ". Request class name: " + this.request.getClassName() - + ". Request method: " + this.request.getMethodName()); - } - return response; - } - - public void done(RpcResponse response) { - this.response = response; - latch.countDown(); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java deleted file mode 100644 index 224ab7b869..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.protocol; - -public enum EventType { - - HEARTBEAT((byte) 1, "heartbeat"), - REQUEST((byte) 2, "business request"), - RESPONSE((byte) 3, "business response"); - - private Byte type; - - private String description; - - EventType(Byte type, String description) { - this.type = type; - this.description = description; - } - - public Byte getType() { - return type; - } - - public String getDescription() { - return description; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java deleted file mode 100644 index d16b5883d3..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.protocol; - -public class MessageHeader { - - private byte version = 1; - - private byte eventType; - - private int msgLength = 0; - - private long requestId = 0L; - - private byte serialization = 0; - - private short magic = RpcProtocolConstants.MAGIC; - - public short getMagic() { - return magic; - } - - public byte getVersion() { - return version; - } - - public void setVersion(byte version) { - this.version = version; - } - - public byte getEventType() { - return eventType; - } - - public void setEventType(byte eventType) { - this.eventType = eventType; - } - - public int getMsgLength() { - return msgLength; - } - - public void setMsgLength(int msgLength) { - this.msgLength = msgLength; - } - - public long getRequestId() { - return requestId; - } - - public void setRequestId(long requestId) { - this.requestId = requestId; - } - - public byte getSerialization() { - return serialization; - } - - public void setSerialization(byte serialization) { - this.serialization = serialization; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java deleted file mode 100644 index 23c3ee8b35..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.remote; - -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.NettyUtils; -import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; -import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; -import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; -import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; -import org.apache.dolphinscheduler.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.rpc.future.RpcFuture; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; - -import java.net.InetSocketAddress; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import lombok.extern.slf4j.Slf4j; -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import io.netty.handler.timeout.IdleStateHandler; - -/** - * NettyClient - */ -@Slf4j -public class NettyClient { - - public static NettyClient getInstance() { - return NettyClient.NettyClientInner.INSTANCE; - } - - private static class NettyClientInner { - - private static final NettyClient INSTANCE = new NettyClient(new NettyClientConfig()); - } - - /** - * worker group - */ - private final EventLoopGroup workerGroup; - - /** - * client config - */ - private final NettyClientConfig clientConfig; - - /** - * client bootstrap - */ - private final Bootstrap bootstrap = new Bootstrap(); - - /** - * started flag - */ - private final AtomicBoolean isStarted = new AtomicBoolean(false); - - /** - * channels - */ - private final ConcurrentHashMap channels = new ConcurrentHashMap<>(128); - - /** - * get channel - */ - private Channel getChannel(Host host) { - Channel channel = channels.get(host); - if (channel != null && channel.isActive()) { - return channel; - } - return createChannel(host, true); - } - - /** - * create channel - * - * @param host host - * @param isSync sync flag - * @return channel - */ - public Channel createChannel(Host host, boolean isSync) { - ChannelFuture future; - try { - synchronized (bootstrap) { - future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); - } - if (isSync) { - future.sync(); - } - if (future.isSuccess()) { - Channel channel = future.channel(); - channels.put(host, channel); - return channel; - } - } catch (Exception ex) { - log.warn(String.format("connect to %s error", host), ex); - } - return null; - } - - /** - * client init - * - * @param clientConfig client config - */ - private NettyClient(final NettyClientConfig clientConfig) { - this.clientConfig = clientConfig; - if (Epoll.isAvailable()) { - this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { - - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); - } - }); - } else { - this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { - - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); - } - }); - } - this.start(); - - } - - /** - * start - */ - private void start() { - - this.bootstrap - .group(this.workerGroup) - .channel(NettyUtils.getSocketChannelClass()) - .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) - .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) - .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) - .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) - .handler(new LoggingHandler(LogLevel.DEBUG)) - .handler(new ChannelInitializer() { - - @Override - public void initChannel(SocketChannel ch) { - ch.pipeline() - .addLast(new NettyEncoder()) - .addLast(new NettyDecoder(RpcResponse.class)) - .addLast("client-idle-handler", - new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, - TimeUnit.MILLISECONDS)) - .addLast(new NettyClientHandler()); - } - }); - - isStarted.compareAndSet(false, true); - } - - public RpcResponse sendMsg(Host host, RpcProtocol protocol, Boolean async) { - - Channel channel = getChannel(host); - assert channel != null; - RpcRequest request = protocol.getBody(); - RpcRequestCache rpcRequestCache = new RpcRequestCache(); - String serviceName = request.getClassName() + request.getMethodName(); - rpcRequestCache.setServiceName(serviceName); - long reqId = protocol.getMsgHeader().getRequestId(); - RpcFuture future = null; - if (Boolean.FALSE.equals(async)) { - future = new RpcFuture(request, reqId); - rpcRequestCache.setRpcFuture(future); - } - RpcRequestTable.put(protocol.getMsgHeader().getRequestId(), rpcRequestCache); - channel.writeAndFlush(protocol); - RpcResponse result = null; - if (Boolean.TRUE.equals(async)) { - result = new RpcResponse(); - result.setStatus((byte) 0); - result.setResult(true); - return result; - } - try { - assert future != null; - result = future.get(); - } catch (InterruptedException e) { - log.error("send msg error，service name is {}", serviceName, e); - Thread.currentThread().interrupt(); - } - return result; - } - - /** - * close - */ - public void close() { - if (isStarted.compareAndSet(true, false)) { - try { - closeChannels(); - if (workerGroup != null) { - this.workerGroup.shutdownGracefully(); - } - } catch (Exception ex) { - log.error("netty client close exception", ex); - } - log.info("netty client closed"); - } - } - - /** - * close channels - */ - private void closeChannels() { - for (Channel channel : this.channels.values()) { - channel.close(); - } - this.channels.clear(); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java deleted file mode 100644 index cbef395cd6..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.remote; - -import org.apache.dolphinscheduler.rpc.client.ConsumerConfig; -import org.apache.dolphinscheduler.rpc.client.ConsumerConfigCache; -import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; -import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; -import org.apache.dolphinscheduler.rpc.future.RpcFuture; -import org.apache.dolphinscheduler.rpc.protocol.EventType; -import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; - -import java.lang.reflect.InvocationTargetException; - -import lombok.extern.slf4j.Slf4j; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleStateEvent; - -/** - * NettyClientHandler - */ -@ChannelHandler.Sharable -@Slf4j -public class NettyClientHandler extends ChannelInboundHandlerAdapter { - - private static final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - ctx.channel().close(); - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { - RpcProtocol rpcProtocol = (RpcProtocol) msg; - - RpcResponse rsp = (RpcResponse) rpcProtocol.getBody(); - long reqId = rpcProtocol.getMsgHeader().getRequestId(); - RpcRequestCache rpcRequest = RpcRequestTable.get(reqId); - - if (null == rpcRequest) { - log.warn("rpc read error,this request does not exist"); - return; - } - threadPoolManager.addExecuteTask(() -> readHandler(rsp, rpcRequest, reqId)); - } - - private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest, long reqId) { - String serviceName = rpcRequest.getServiceName(); - ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); - if (Boolean.FALSE.equals(consumerConfig.getAsync())) { - RpcFuture future = rpcRequest.getRpcFuture(); - RpcRequestTable.remove(reqId); - future.done(rsp); - return; - } - - if (Boolean.FALSE.equals(consumerConfig.getCallBack())) { - return; - } - - if (rsp.getStatus() == 0) { - - try { - consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); - } catch (InstantiationException | IllegalAccessException | NoSuchMethodException - | InvocationTargetException e) { - log.error("rpc service call back error,serviceName {},rsp {}", serviceName, rsp); - } - } else { - log.error("rpc response error ,serviceName {},rsp {}", serviceName, rsp); - } - - } - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - - if (evt instanceof IdleStateEvent) { - RpcProtocol rpcProtocol = new RpcProtocol(); - MessageHeader messageHeader = new MessageHeader(); - messageHeader.setEventType(EventType.HEARTBEAT.getType()); - rpcProtocol.setMsgHeader(messageHeader); - ctx.channel().writeAndFlush(rpcProtocol); - log.debug("send heart beat msg..."); - } else { - super.userEventTriggered(ctx, evt); - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - log.error("exceptionCaught : {}", cause.getMessage(), cause); - ctx.channel().close(); - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java deleted file mode 100644 index e55ad79537..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.remote; - -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.NettyUtils; -import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; -import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; -import org.apache.dolphinscheduler.rpc.common.RpcRequest; - -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import lombok.extern.slf4j.Slf4j; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import io.netty.handler.timeout.IdleStateHandler; - -/** - * NettyServer - */ -@Slf4j -public class NettyServer { - - /** - * boss group - */ - private final EventLoopGroup bossGroup; - - /** - * worker group - */ - private final EventLoopGroup workGroup; - - /** - * server config - */ - private final NettyServerConfig serverConfig; - - /** - * server bootstrap - */ - private final ServerBootstrap serverBootstrap = new ServerBootstrap(); - - /** - * started flag - */ - private final AtomicBoolean isStarted = new AtomicBoolean(false); - - /** - * server init - * - * @param serverConfig server config - */ - public NettyServer(final NettyServerConfig serverConfig) { - this.serverConfig = serverConfig; - if (Epoll.isAvailable()) { - this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() { - - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); - } - }); - - this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { - - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, - String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); - } - }); - } else { - this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { - - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); - } - }); - - this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { - - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, - String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); - } - }); - } - this.start(); - } - - /** - * server start - */ - public void start() { - if (isStarted.compareAndSet(false, true)) { - this.serverBootstrap - .group(this.bossGroup, this.workGroup) - .channel(NettyUtils.getServerSocketChannelClass()) - .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) - .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) - .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) - .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) - .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) - .handler(new LoggingHandler(LogLevel.DEBUG)) - .childHandler(new ChannelInitializer() { - - @Override - protected void initChannel(SocketChannel ch) { - initNettyChannel(ch); - } - }); - - ChannelFuture future; - try { - future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); - } catch (Exception e) { - log.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); - throw new RuntimeException( - String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); - } - if (future.isSuccess()) { - log.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); - } else if (future.cause() != null) { - throw new RuntimeException( - String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), - future.cause()); - } else { - throw new RuntimeException( - String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); - } - } - } - - /** - * init netty channel - * - * @param ch socket channel - */ - private void initNettyChannel(SocketChannel ch) { - ch.pipeline() - .addLast(new NettyDecoder(RpcRequest.class)) - .addLast(new NettyEncoder()) - .addLast("server-idle-handle", - new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) - .addLast("handler", new NettyServerHandler()); - } - - public void close() { - if (isStarted.compareAndSet(true, false)) { - try { - if (bossGroup != null) { - this.bossGroup.shutdownGracefully(); - } - if (workGroup != null) { - this.workGroup.shutdownGracefully(); - } - - } catch (Exception ex) { - log.error("netty server close exception", ex); - } - log.info("netty server closed"); - } - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java deleted file mode 100644 index ba3a8b5fcf..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.remote; - -import org.apache.dolphinscheduler.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; -import org.apache.dolphinscheduler.rpc.config.ServiceBean; -import org.apache.dolphinscheduler.rpc.protocol.EventType; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; - -import java.lang.reflect.Method; - -import lombok.extern.slf4j.Slf4j; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleStateEvent; - -/** - * NettyServerHandler - */ -@Slf4j -public class NettyServerHandler extends ChannelInboundHandlerAdapter { - - private static final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - log.info("channel close"); - ctx.channel().close(); - } - - @Override - public void channelActive(ChannelHandlerContext ctx) { - log.info("client connect success !" + ctx.channel().remoteAddress()); - } - - @Override - @SuppressWarnings("unchecked") - public void channelRead(ChannelHandlerContext ctx, Object msg) { - RpcProtocol rpcProtocol = (RpcProtocol) msg; - if (rpcProtocol.getMsgHeader().getEventType() == EventType.HEARTBEAT.getType()) { - log.info("heart beat"); - return; - } - threadPoolManager.addExecuteTask(() -> readHandler(ctx, rpcProtocol)); - } - - private void readHandler(ChannelHandlerContext ctx, RpcProtocol protocol) { - RpcRequest req = (RpcRequest) protocol.getBody(); - RpcResponse response = new RpcResponse(); - - response.setStatus((byte) 0); - - String classname = req.getClassName(); - - String methodName = req.getMethodName(); - - Class[] parameterTypes = req.getParameterTypes(); - - Object[] arguments = req.getParameters(); - Object result = null; - try { - Class serviceClass = ServiceBean.getServiceClass(classname); - - Object object = serviceClass.newInstance(); - - Method method = serviceClass.getMethod(methodName, parameterTypes); - - result = method.invoke(object, arguments); - } catch (Exception e) { - log.error("netty server execute error,service name :{} method name :{} ", classname + methodName, e); - response.setStatus((byte) -1); - } - - response.setResult(result); - protocol.setBody(response); - protocol.getMsgHeader().setEventType(EventType.RESPONSE.getType()); - ctx.writeAndFlush(protocol); - } - - @Override - public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof IdleStateEvent) { - log.debug("IdleStateEvent triggered, send heartbeat to channel " + ctx.channel()); - } else { - super.userEventTriggered(ctx, evt); - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - log.error("exceptionCaught : {}", cause.getMessage(), cause); - ctx.channel().close(); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java deleted file mode 100644 index ebbc7962a2..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.serializer; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.protostuff.LinkedBuffer; -import io.protostuff.ProtostuffIOUtil; -import io.protostuff.Schema; -import io.protostuff.runtime.RuntimeSchema; - -public class ProtoStuffSerializer implements Serializer { - - private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); - - private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); - - @SuppressWarnings("unchecked") - private static Schema getSchema(Class clazz) { - return (Schema) schemaCache.computeIfAbsent(clazz, RuntimeSchema::createFrom); - } - - @Override - public byte[] serialize(T obj) { - Class clazz = (Class) obj.getClass(); - Schema schema = getSchema(clazz); - byte[] data; - try { - data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); - } finally { - buffer.clear(); - } - return data; - } - - @Override - public T deserialize(byte[] data, Class clz) { - Schema schema = getSchema(clz); - T obj = schema.newMessage(); - if (null == obj) { - return null; - } - ProtostuffIOUtil.mergeFrom(data, obj, schema); - return obj; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java deleted file mode 100644 index ef2a8846f1..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.serializer; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import io.protostuff.LinkedBuffer; -import io.protostuff.ProtostuffIOUtil; -import io.protostuff.Schema; -import io.protostuff.runtime.RuntimeSchema; - -/** - * ProtoStuffUtils - */ -public class ProtoStuffUtils { - - private ProtoStuffUtils() { - throw new IllegalStateException("Utility class"); - } - - private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); - - private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); - - @SuppressWarnings("unchecked") - public static byte[] serialize(T obj) { - Class clazz = (Class) obj.getClass(); - Schema schema = getSchema(clazz); - byte[] data; - try { - data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); - } finally { - buffer.clear(); - } - return data; - } - - @SuppressWarnings("unchecked") - private static Schema getSchema(Class clazz) { - return (Schema) schemaCache.computeIfAbsent(clazz, RuntimeSchema::createFrom); - } - - public static T deserialize(byte[] bytes, Class clazz) { - Schema schema = getSchema(clazz); - T obj = schema.newMessage(); - if (null == obj) { - return null; - } - ProtostuffIOUtil.mergeFrom(bytes, obj, schema); - return obj; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java deleted file mode 100644 index a826560d03..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc.serializer; - -import java.util.HashMap; - -public enum RpcSerializer { - - PROTOSTUFF((byte) 1, new ProtoStuffSerializer()); - - byte type; - - Serializer serializer; - - RpcSerializer(byte type, Serializer serializer) { - this.type = type; - this.serializer = serializer; - } - - public byte getType() { - return type; - } - - private static HashMap SERIALIZERS_MAP = new HashMap<>(); - - static { - for (RpcSerializer rpcSerializer : RpcSerializer.values()) { - SERIALIZERS_MAP.put(rpcSerializer.type, rpcSerializer.serializer); - } - } - - public static Serializer getSerializerByType(byte type) { - return SERIALIZERS_MAP.get(type); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java deleted file mode 100644 index 3c44e490b1..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote; - -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.future.InvokeCallback; -import org.apache.dolphinscheduler.remote.future.ResponseFuture; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.remote.utils.Host; - -import java.io.Serializable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicLong; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; - -/** - * netty remote client test - */ -public class NettyRemotingClientTest { - - /** - * test send sync - */ - @Test - public void testSendSync() { - NettyServerConfig serverConfig = new NettyServerConfig(); - - NettyRemotingServer server = new NettyRemotingServer(serverConfig); - server.registerProcessor(new NettyRequestProcessor() { - - @Override - public void process(Channel channel, Message message) { - channel.writeAndFlush(Pong.create(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.PING; - } - }); - - server.start(); - // - final NettyClientConfig clientConfig = new NettyClientConfig(); - NettyRemotingClient client = new NettyRemotingClient(clientConfig); - Message messagePing = Ping.create(); - try { - Message response = client.sendSync(new Host("127.0.0.1", serverConfig.getListenPort()), messagePing, 2000); - Assertions.assertEquals(messagePing.getOpaque(), response.getOpaque()); - } catch (Exception e) { - e.printStackTrace(); - } - server.close(); - client.close(); - } - - /** - * test sned async - */ - @Test - public void testSendAsync() { - NettyServerConfig serverConfig = new NettyServerConfig(); - - NettyRemotingServer server = new NettyRemotingServer(serverConfig); - server.registerProcessor(new NettyRequestProcessor() { - - @Override - public void process(Channel channel, Message message) { - channel.writeAndFlush(Pong.create(message.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.PING; - } - }); - server.start(); - // - final NettyClientConfig clientConfig = new NettyClientConfig(); - NettyRemotingClient client = new NettyRemotingClient(clientConfig); - CountDownLatch latch = new CountDownLatch(1); - Message messagePing = Ping.create(); - try { - final AtomicLong opaque = new AtomicLong(0); - client.sendAsync(new Host("127.0.0.1", serverConfig.getListenPort()), messagePing, 2000, - new InvokeCallback() { - - @Override - public void operationComplete(ResponseFuture responseFuture) { - opaque.set(responseFuture.getOpaque()); - latch.countDown(); - } - }); - latch.await(); - Assertions.assertEquals(messagePing.getOpaque(), opaque.get()); - } catch (Exception e) { - e.printStackTrace(); - } - server.close(); - client.close(); - } - - private static class Ping implements Serializable { - - /** - * ping body - */ - protected static final ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; - - /** - * request command body - */ - private static final byte[] EMPTY_BODY_ARRAY = new byte[0]; - - private static final ByteBuf PING_BUF; - - static { - ByteBuf ping = Unpooled.buffer(); - ping.writeByte(Message.MAGIC); - ping.writeByte(MessageType.PING.ordinal()); - ping.writeLong(0); - ping.writeInt(0); - ping.writeBytes(EMPTY_BODY); - PING_BUF = Unpooled.unreleasableBuffer(ping).asReadOnly(); - } - - /** - * ping content - * - * @return result - */ - public static ByteBuf pingContent() { - return PING_BUF.duplicate(); - } - - /** - * create ping command - * - * @return command - */ - public static Message create() { - Message message = new Message(); - message.setType(MessageType.PING); - message.setBody(EMPTY_BODY_ARRAY); - return message; - } - } - - private static class Pong implements Serializable { - - /** - * pong body - */ - protected static final ByteBuf EMPTY_BODY = Unpooled.EMPTY_BUFFER; - - /** - * pong command body - */ - private static final byte[] EMPTY_BODY_ARRAY = new byte[0]; - - /** - * pong byte buffer - */ - private static final ByteBuf PONG_BUF; - - static { - ByteBuf ping = Unpooled.buffer(); - ping.writeByte(Message.MAGIC); - ping.writeByte(MessageType.PONG.ordinal()); - ping.writeLong(0); - ping.writeInt(0); - ping.writeBytes(EMPTY_BODY); - PONG_BUF = Unpooled.unreleasableBuffer(ping).asReadOnly(); - } - - /** - * pong content - * - * @return result - */ - public static ByteBuf pongContent() { - return PONG_BUF.duplicate(); - } - - /** - * package pong command - * - * @param opaque request unique identification - * @return command - */ - public static Message create(long opaque) { - Message message = new Message(opaque); - message.setType(MessageType.PONG); - message.setBody(EMPTY_BODY_ARRAY); - return message; - } - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestTest.java deleted file mode 100644 index affc08028d..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.alert; - -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class AlertSendRequestTest { - - @Test - public void testConvert2Command() { - int groupId = 1; - String title = "test-title"; - String content = "test-content"; - AlertSendRequest requestCommand = - new AlertSendRequest(groupId, title, content, WarningType.FAILURE.getCode()); - Message message = requestCommand.convert2Command(); - Assertions.assertEquals(MessageType.ALERT_SEND_REQUEST, message.getType()); - AlertSendRequest verifyCommand = new AlertSendRequest(); - verifyCommand.setGroupId(groupId); - verifyCommand.setContent(content); - verifyCommand.setTitle(title); - - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseTest.java deleted file mode 100644 index cd6fab8421..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.alert; - -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class AlertSendResponseTest { - - @Test - public void testConvert2Command() { - AlertSendResponse alertSendResponse = new AlertSendResponse(); - alertSendResponse.setSuccess(false); - List responseResults = new ArrayList<>(); - AlertSendResponse.AlertSendResponseResult responseResult1 = new AlertSendResponse.AlertSendResponseResult(); - responseResult1.setSuccess(false); - responseResult1.setMessage("fail"); - responseResults.add(responseResult1); - - AlertSendResponse.AlertSendResponseResult responseResult2 = - new AlertSendResponse.AlertSendResponseResult(true, "success"); - responseResults.add(responseResult2); - alertSendResponse.setResResults(responseResults); - - Message message = alertSendResponse.convert2Command(1); - Assertions.assertEquals(MessageType.RESPONSE, message.getType()); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequestTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequestTest.java deleted file mode 100644 index a9f032b6c1..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequestTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.cache; - -import org.apache.dolphinscheduler.common.enums.CacheType; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class CacheExpireRequestTest { - - @Test - public void testConvert2Command() { - CacheExpireRequest cacheExpireRequest = new CacheExpireRequest(CacheType.TENANT, "1"); - Message message = cacheExpireRequest.convert2Command(); - Assertions.assertEquals(MessageType.CACHE_EXPIRE, message.getType()); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java deleted file mode 100644 index 3cb6fb319c..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.future; - -import org.apache.dolphinscheduler.remote.future.InvokeCallback; -import org.apache.dolphinscheduler.remote.future.ResponseFuture; -import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ResponseFutureTest { - - @Test - public void testScanFutureTable() { - ScheduledExecutorService executorService = - Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service")); - executorService.scheduleAtFixedRate(new Runnable() { - - @Override - public void run() { - ResponseFuture.scanFutureTable(); - } - }, 3000, 1000, TimeUnit.MILLISECONDS); - - CountDownLatch latch = new CountDownLatch(1); - InvokeCallback invokeCallback = new InvokeCallback() { - - @Override - public void operationComplete(ResponseFuture responseFuture) { - latch.countDown(); - } - }; - ResponseFuture future = new ResponseFuture(1, 2000, invokeCallback, null); - try { - latch.await(5000, TimeUnit.MILLISECONDS); - Assertions.assertNull(ResponseFuture.getFuture(1)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - executorService.shutdownNow(); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestTest.java deleted file mode 100644 index b66066200f..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class GetLogBytesRequestTest { - - @Test - public void testConvert2Command() { - GetLogBytesRequest getLogBytesRequest = new GetLogBytesRequest("/opt/test"); - Message message = getLogBytesRequest.convert2Command(); - Assertions.assertEquals(MessageType.GET_LOG_BYTES_REQUEST, message.getType()); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseTest.java deleted file mode 100644 index 3c0e17362c..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.Message; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class GetLogBytesResponseTest { - - private byte[] data; - - @Test - public void testConvert2Command() { - GetLogBytesResponse getLogBytesResponse = new GetLogBytesResponse(data); - Message message = getLogBytesResponse.convert2Command(122); - Assertions.assertNotNull(message); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestTest.java deleted file mode 100644 index 6cb18ebf05..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.Message; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class RemoveTaskLogRequestTest { - - @Test - public void testConvert2Command() { - RemoveTaskLogResponse removeTaskLogResponse = new RemoveTaskLogResponse(); - removeTaskLogResponse.setStatus(true); - Message message = removeTaskLogResponse.convert2Command(122); - Assertions.assertNotNull(message); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseTest.java deleted file mode 100644 index b9b16b0368..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.Message; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class RemoveTaskLogResponseTest { - - @Test - public void testConvert2Command() { - RemoveTaskLogRequest removeTaskLogRequest = new RemoveTaskLogRequest(); - removeTaskLogRequest.setPath("/opt/zhangsan"); - Message message = removeTaskLogRequest.convert2Command(); - Assertions.assertNotNull(message); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestTest.java deleted file mode 100644 index 10e230bac4..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.command.log; - -import org.apache.dolphinscheduler.remote.command.Message; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class RollViewLogRequestTest { - - @Test - public void testConvert2Command() { - RollViewLogRequest rollViewLogRequest = new RollViewLogRequest(); - rollViewLogRequest.setPath("/opt/test"); - rollViewLogRequest.setSkipLineNum(1); - rollViewLogRequest.setLimit(1); - Message message = rollViewLogRequest.convert2Command(); - Assertions.assertNotNull(message); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java deleted file mode 100644 index ce8a433a4b..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.utils; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * host test - */ -public class HostTest { - - @Test - public void testHost() { - Host host = Host.of("192.158.2.2:22"); - Assertions.assertEquals(22, host.getPort()); - host = new Host("127.0.0.1:8888"); - Assertions.assertEquals("127.0.0.1", host.getIp()); - Assertions.assertEquals(8888, host.getPort()); - - host = new Host("2001:db8:1::ab9:C0A8:102:5678"); - Assertions.assertEquals("2001:db8:1::ab9:C0A8:102", host.getIp()); - Assertions.assertEquals(5678, host.getPort()); - } - -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java deleted file mode 100644 index 7beee00106..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc; - -import org.apache.dolphinscheduler.rpc.base.Rpc; - -/** - * IUserService - */ -public interface IUserService { - - @Rpc(async = true, serviceCallback = UserCallback.class) - Boolean say(String s); - - Integer hi(int num); - - @Rpc(async = true) - Boolean callBackIsFalse(String s); -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java deleted file mode 100644 index 3ff50d666a..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc; - -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.rpc.client.IRpcClient; -import org.apache.dolphinscheduler.rpc.client.RpcClient; -import org.apache.dolphinscheduler.rpc.remote.NettyClient; -import org.apache.dolphinscheduler.rpc.remote.NettyServer; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -public class RpcTest { - - private NettyServer nettyServer; - - private IUserService userService; - - private Host host; - - @BeforeEach - public void before() throws Exception { - nettyServer = new NettyServer(new NettyServerConfig()); - IRpcClient rpcClient = new RpcClient(); - host = new Host("127.0.0.1", 12346); - userService = rpcClient.create(IUserService.class, host); - } - - @AfterEach - public void after() { - NettyClient.getInstance().close(); - nettyServer.close(); - } - -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java deleted file mode 100644 index 72c3c705ed..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc; - -import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * UserCallback - */ -public class UserCallback extends AbstractRpcCallBack { - - private static final Logger logger = LoggerFactory.getLogger(UserCallback.class); - - @Override - public void run(Object object) { - String msg = (String) object; - logger.debug("Kris---------------------------------userCallBack msg is {}", msg); - } -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java deleted file mode 100644 index 80fb69678a..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc; - -import org.apache.dolphinscheduler.rpc.base.RpcService; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * UserService - */ -@RpcService("IUserService") -public class UserService implements IUserService { - - private static final Logger logger = LoggerFactory.getLogger(UserService.class); - - @Override - public Boolean say(String s) { - - logger.info("Kris UserService say-------------------------------Synchronous call msg{}", s); - return true; - } - - @Override - public Integer hi(int num) { - - logger.info("Kris UserService hi-------------------------------async call msg{}", num); - return ++num; - } - - @Override - public Boolean callBackIsFalse(String s) { - logger.info("Kris UserService callBackIsFalse-------------------------------async call msg{}", s); - return null; - } -} diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 18995ecdf0..0bca706c3b 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -41,11 +41,7 @@ - - - org.apache.dolphinscheduler - dolphinscheduler-remote - + org.apache.dolphinscheduler dolphinscheduler-dao @@ -63,6 +59,16 @@ dolphinscheduler-task-api + + org.apache.dolphinscheduler + dolphinscheduler-extract-master + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-worker + + com.cronutils cron-utils diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java deleted file mode 100644 index 34687ec6eb..0000000000 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.alert; - -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequest; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse; -import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.JsonSerializer; - -import java.util.concurrent.atomic.AtomicBoolean; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AlertClientService implements AutoCloseable { - - private final NettyRemotingClient client; - - private final AtomicBoolean isRunning; - - private String host; - - private int port; - - /** - * request time out - */ - private static final long ALERT_REQUEST_TIMEOUT = 10 * 1000L; - - /** - * alert client - */ - public AlertClientService() { - this.client = NettyRemotingClientFactory.buildNettyRemotingClient(); - this.isRunning = new AtomicBoolean(true); - } - - /** - * close - */ - @Override - public void close() { - if (isRunning.compareAndSet(true, false)) { - log.warn("Alert client is already closed"); - return; - } - - log.info("Alter client closing"); - this.client.close(); - log.info("Alter client closed"); - } - - /** - * alert sync send data - * @param groupId - * @param title - * @param content - * @return - */ - public AlertSendResponse sendAlert(int groupId, String title, String content, int strategy) { - return this.sendAlert(this.host, this.port, groupId, title, content, strategy); - } - - /** - * alert sync send data - * @param host host - * @param port port - * @param groupId groupId - * @param title title - * @param content content - * @return AlertSendResponseCommand - */ - public AlertSendResponse sendAlert(String host, int port, int groupId, String title, String content, - int strategy) { - log.info("sync alert send, host : {}, port : {}, groupId : {}, title : {} , strategy : {} ", host, port, - groupId, title, strategy); - AlertSendRequest request = new AlertSendRequest(groupId, title, content, strategy); - final Host address = new Host(host, port); - try { - Message message = request.convert2Command(); - Message response = this.client.sendSync(address, message, ALERT_REQUEST_TIMEOUT); - if (response != null) { - return JsonSerializer.deserialize(response.getBody(), AlertSendResponse.class); - } - } catch (Exception e) { - log.error("sync alert send error", e); - } finally { - this.client.closeChannel(address); - } - return null; - } - - public boolean isRunning() { - return isRunning.get(); - } -} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheNotifyServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheNotifyServiceImpl.java deleted file mode 100644 index f48820dc80..0000000000 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/impl/CacheNotifyServiceImpl.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.cache.impl; - -import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.registry.api.RegistryClient; -import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.service.cache.CacheNotifyService; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import io.netty.channel.Channel; - -/** - * cache notify service - */ -@Service -@Slf4j -public class CacheNotifyServiceImpl implements CacheNotifyService { - - @Autowired - private RegistryClient registryClient; - - /** - * remote channels - */ - private static final ConcurrentHashMap REMOTE_CHANNELS = new ConcurrentHashMap<>(); - - /** - * netty remoting client - */ - private final NettyRemotingClient nettyRemotingClient; - - public CacheNotifyServiceImpl() { - final NettyClientConfig clientConfig = new NettyClientConfig(); - this.nettyRemotingClient = new NettyRemotingClient(clientConfig); - } - - /** - * add channel - * - * @param channel channel - */ - private void cache(Host host, NettyRemoteChannel channel) { - REMOTE_CHANNELS.put(host, channel); - } - - /** - * remove channel - */ - private void remove(Host host) { - REMOTE_CHANNELS.remove(host); - } - - /** - * get remote channel - * - * @return netty remote channel - */ - private NettyRemoteChannel getRemoteChannel(Host host) { - NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(host); - if (nettyRemoteChannel != null) { - if (nettyRemoteChannel.isActive()) { - return nettyRemoteChannel; - } else { - this.remove(host); - } - } - - Channel channel = nettyRemotingClient.getChannel(host); - if (channel == null) { - return null; - } - - NettyRemoteChannel remoteChannel = new NettyRemoteChannel(channel); - this.cache(host, remoteChannel); - return remoteChannel; - } - - /** - * send result to master - * - * @param message command - */ - @Override - public void notifyMaster(Message message) { - log.info("send result, command:{}", message.toString()); - try { - List serverList = registryClient.getServerList(RegistryNodeType.MASTER); - if (CollectionUtils.isEmpty(serverList)) { - return; - } - - for (Server server : serverList) { - Host host = new Host(server.getHost(), server.getPort()); - NettyRemoteChannel nettyRemoteChannel = getRemoteChannel(host); - if (nettyRemoteChannel == null) { - continue; - } - nettyRemoteChannel.writeAndFlush(message); - } - } catch (Exception e) { - log.error("notify master error", e); - } - } -} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java deleted file mode 100644 index fb71991469..0000000000 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.log; - -import static org.apache.dolphinscheduler.common.constants.Constants.APPID_COLLECT; -import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_COLLECT_WAY; -import static org.apache.dolphinscheduler.common.utils.LogUtils.readWholeFileContentFromLocal; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.log.GetAppIdRequest; -import org.apache.dolphinscheduler.remote.command.log.GetAppIdResponse; -import org.apache.dolphinscheduler.remote.command.log.GetLogBytesRequest; -import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponse; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequest; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponse; -import org.apache.dolphinscheduler.remote.command.log.RollViewLogRequest; -import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponse; -import org.apache.dolphinscheduler.remote.command.log.ViewLogRequest; -import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseResponse; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; -import org.apache.dolphinscheduler.remote.utils.Host; - -import java.util.List; - -import javax.annotation.Nullable; - -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Service; - -@Service -@Slf4j -public class LogClient implements AutoCloseable { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private final NettyRemotingClient client; - - private static final long LOG_REQUEST_TIMEOUT = 10 * 1000L; - - public LogClient() { - client = NettyRemotingClientFactory.buildNettyRemotingClient(); - } - - /** - * roll view log - * - * @param host host - * @param port port - * @param path path - * @param skipLineNum skip line number - * @param limit limit - * @return log content - */ - public String rollViewLog(String host, int port, String path, int skipLineNum, int limit) { - log.info("Roll view log from host : {}, port : {}, path {}, skipLineNum {} ,limit {}", host, port, path, - skipLineNum, limit); - RollViewLogRequest request = new RollViewLogRequest(path, skipLineNum, limit); - final Host address = new Host(host, port); - try { - Message message = request.convert2Command(); - Message response = client.sendSync(address, message, LOG_REQUEST_TIMEOUT); - if (response != null) { - RollViewLogResponse rollReviewLog = - JSONUtils.parseObject(response.getBody(), RollViewLogResponse.class); - return rollReviewLog.getMsg(); - } - return "Roll view log response is null"; - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - log.error( - "Roll view log from host : {}, port : {}, path {}, skipLineNum {} ,limit {} error, the current thread has been interrupted", - host, port, path, skipLineNum, limit, ex); - return "Roll view log error: " + ex.getMessage(); - } catch (Exception e) { - log.error("Roll view log from host : {}, port : {}, path {}, skipLineNum {} ,limit {} error", host, port, - path, skipLineNum, limit, e); - return "Roll view log error: " + e.getMessage(); - } - } - - /** - * view log - * - * @param host host - * @param port port - * @param path path - * @return log content - */ - public String viewLog(String host, int port, String path) { - log.info("View log from host: {}, port: {}, logPath: {}", host, port, path); - ViewLogRequest request = new ViewLogRequest(path); - final Host address = new Host(host, port); - try { - if (NetUtils.getHost().equals(host)) { - return readWholeFileContentFromLocal(request.getPath()); - } else { - Message message = request.convert2Command(); - Message response = this.client.sendSync(address, message, LOG_REQUEST_TIMEOUT); - if (response != null) { - ViewLogResponseResponse viewLog = - JSONUtils.parseObject(response.getBody(), ViewLogResponseResponse.class); - return viewLog.getMsg(); - } - return "View log response is null"; - } - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - log.error("View log from host: {}, port: {}, logPath: {} error, the current thread has been interrupted", - host, port, path, ex); - return "View log error: " + ex.getMessage(); - } catch (Exception e) { - log.error("View log from host: {}, port: {}, logPath: {} error", host, port, path, e); - return "View log error: " + e.getMessage(); - } - } - - /** - * get log size - * - * @param host host - * @param port port - * @param path log path - * @return log content bytes - */ - public byte[] getLogBytes(String host, int port, String path) { - log.info("Get log bytes from host: {}, port: {}, logPath {}", host, port, path); - GetLogBytesRequest request = new GetLogBytesRequest(path); - final Host address = new Host(host, port); - try { - Message message = request.convert2Command(); - Message response = this.client.sendSync(address, message, LOG_REQUEST_TIMEOUT); - if (response != null) { - GetLogBytesResponse getLog = - JSONUtils.parseObject(response.getBody(), GetLogBytesResponse.class); - return getLog.getData() == null ? EMPTY_BYTE_ARRAY : getLog.getData(); - } - return EMPTY_BYTE_ARRAY; - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - log.error( - "Get logSize from host: {}, port: {}, logPath: {} error, the current thread has been interrupted", - host, port, path, ex); - return EMPTY_BYTE_ARRAY; - } catch (Exception e) { - log.error("Get logSize from host: {}, port: {}, logPath: {} error", host, port, path, e); - return EMPTY_BYTE_ARRAY; - } - } - - /** - * remove task log - * - * @param host host - * @param path path - */ - public void removeTaskLog(@NonNull Host host, String path) { - log.info("Begin remove task log from host: {} logPath {}", host, path); - RemoveTaskLogRequest request = new RemoveTaskLogRequest(path); - try { - Message message = request.convert2Command(); - client.sendAsync(host, message, LOG_REQUEST_TIMEOUT, responseFuture -> { - if (responseFuture.getCause() != null) { - log.error("Remove task log from host: {} logPath {} error, meet an unknown exception", host, - path, responseFuture.getCause()); - return; - } - Message response = responseFuture.getResponseCommand(); - if (response == null) { - log.error("Remove task log from host: {} logPath {} error, response is null", host, path); - return; - } - RemoveTaskLogResponse removeTaskLogResponse = - JSONUtils.parseObject(response.getBody(), RemoveTaskLogResponse.class); - if (removeTaskLogResponse.getStatus()) { - log.info("Success remove task log from host: {} logPath {}", host, path); - } else { - log.error("Remove task log from host: {} logPath {} error", host, path); - } - }); - } catch (InterruptedException interruptedException) { - Thread.currentThread().interrupt(); - log.error("Remove task log from host: {} logPath {} error, the current thread has been interrupted", - host, - path, interruptedException); - } catch (Exception e) { - log.error("Remove task log from host: {}, logPath: {} error", host, path, e); - } - } - - public @Nullable List getAppIds(@NonNull String host, int port, String taskLogFilePath, - String taskAppInfoPath, - int taskInstanceId) throws RemotingException, InterruptedException { - log.info("Begin to get appIds from worker: {}:{} taskLogPath: {}, taskAppInfoPath: {}", host, port, - taskLogFilePath, taskAppInfoPath); - final Host workerAddress = new Host(host, port); - List appIds = null; - if (NetUtils.getHost().equals(host)) { - appIds = LogUtils.getAppIds(taskLogFilePath, taskAppInfoPath, - PropertyUtils.getString(APPID_COLLECT, DEFAULT_COLLECT_WAY)); - } else { - final Message message = new GetAppIdRequest(taskInstanceId, taskLogFilePath).convert2Command(); - Message response = this.client.sendSync(workerAddress, message, LOG_REQUEST_TIMEOUT); - if (response != null) { - GetAppIdResponse responseCommand = - JSONUtils.parseObject(response.getBody(), GetAppIdResponse.class); - appIds = responseCommand.getAppIds(); - } - } - log.info("Get appIds: {} from worker: {}:{} taskLogPath: {}, taskAppInfoPath: {}", appIds, host, port, - taskLogFilePath, taskAppInfoPath); - return appIds; - } - - @Override - public void close() { - this.client.close(); - log.info("LogClientService closed"); - } - -} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index 5728ca0779..82bbef30de 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -115,6 +115,11 @@ import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao; import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao; import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao; import org.apache.dolphinscheduler.dao.utils.DqRuleUtils; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.IMasterLogService; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent; +import org.apache.dolphinscheduler.extract.worker.IWorkerLogService; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -125,15 +130,12 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; -import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskUtils; import org.apache.dolphinscheduler.service.command.CommandService; import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.expand.CuringParamsService; -import org.apache.dolphinscheduler.service.log.LogClient; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.utils.ClusterConfUtils; import org.apache.dolphinscheduler.service.utils.DagHelper; @@ -270,9 +272,6 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private ProcessTaskRelationLogMapper processTaskRelationLogMapper; - @Autowired - StateEventCallbackService stateEventCallbackService; - @Autowired private EnvironmentMapper environmentMapper; @@ -294,9 +293,6 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private CuringParamsService curingGlobalParamsService; - @Autowired - private LogClient logClient; - @Autowired private CommandService commandService; @@ -391,12 +387,14 @@ public class ProcessServiceImpl implements ProcessService { boolean update = processInstanceDao.updateById(info); // determine whether the process is normal if (update) { - WorkflowStateEventChangeRequest workflowStateEventChangeRequest = - new WorkflowStateEventChangeRequest( - info.getId(), 0, info.getState(), info.getId(), 0); try { - Host host = new Host(info.getHost()); - stateEventCallbackService.sendResult(host, workflowStateEventChangeRequest.convert2Command()); + final ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(info.getHost(), ITaskInstanceExecutionEventListener.class); + final WorkflowInstanceStateChangeEvent workflowInstanceStateChangeEvent = + new WorkflowInstanceStateChangeEvent(info.getId(), 0, info.getState(), info.getId(), 0); + iTaskInstanceExecutionEventListener + .onWorkflowInstanceInstanceStateChange(workflowInstanceStateChangeEvent); } catch (Exception e) { log.error("sendResultError", e); } @@ -517,7 +515,15 @@ public class ProcessServiceImpl implements ProcessService { if (StringUtils.isEmpty(taskInstance.getHost()) || StringUtils.isEmpty(taskLogPath)) { continue; } - logClient.removeTaskLog(Host.of(taskInstance.getHost()), taskLogPath); + if (TaskUtils.isLogicTask(taskInstance.getTaskType())) { + IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IMasterLogService.class); + masterLogService.removeLogicTaskInstanceLog(taskLogPath); + } else { + IWorkerLogService iWorkerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstance.getHost(), IWorkerLogService.class); + iWorkerLogService.removeTaskInstanceLog(taskLogPath); + } } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java index b2c32a2ac3..71f224334b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java @@ -23,8 +23,6 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.service.log.LogClient; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -162,16 +160,10 @@ public class ProcessUtils { * @param taskExecutionContext taskExecutionContext * @return yarn application ids */ - public static @Nullable List killApplication(@NonNull LogClient logClient, + public static @Nullable List killApplication(@NonNull List appIds, @NonNull TaskExecutionContext taskExecutionContext) { - if (taskExecutionContext.getLogPath() == null) { - return Collections.emptyList(); - } try { Thread.sleep(Constants.SLEEP_TIME_MILLIS); - Host host = Host.of(taskExecutionContext.getHost()); - List appIds = logClient.getAppIds(host.getIp(), host.getPort(), taskExecutionContext.getLogPath(), - taskExecutionContext.getAppInfoPath(), taskExecutionContext.getTaskInstanceId()); if (CollectionUtils.isNotEmpty(appIds)) { taskExecutionContext.setAppIds(String.join(TaskConstants.COMMA, appIds)); if (StringUtils.isEmpty(taskExecutionContext.getExecutePath())) { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java deleted file mode 100644 index 7bd1b26955..0000000000 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.alert; - -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequest; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse; -import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; -import org.apache.dolphinscheduler.remote.utils.Host; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ExtendWith(MockitoExtension.class) -public class AlertClientServiceTest { - - private static final Logger logger = LoggerFactory.getLogger(AlertClientServiceTest.class); - - private NettyRemotingClient client; - - private AlertClientService alertClient; - - private MockedStatic mockedNettyRemotingClientFactory; - - @BeforeEach - public void before() throws Exception { - client = Mockito.mock(NettyRemotingClient.class); - mockedNettyRemotingClientFactory = Mockito.mockStatic(NettyRemotingClientFactory.class); - mockedNettyRemotingClientFactory.when(NettyRemotingClientFactory::buildNettyRemotingClient) - .thenReturn(client); - alertClient = new AlertClientService(); - } - - @AfterEach - public void after() { - mockedNettyRemotingClientFactory.close(); - } - - @Test - public void testSendAlert() throws Exception { - String host = "127.0.0.1"; - int port = 50501; - int groupId = 1; - String title = "test-title"; - String content = "test-content"; - - // 1.alter server does not exist - AlertSendResponse alertSendResponse = - alertClient.sendAlert(host, port, groupId, title, content, WarningType.FAILURE.getCode()); - Assertions.assertNull(alertSendResponse); - - AlertSendRequest alertSendRequest = - new AlertSendRequest(groupId, title, content, WarningType.FAILURE.getCode()); - Message reqMessage = alertSendRequest.convert2Command(); - boolean sendResponseStatus; - List sendResponseResults = new ArrayList<>(); - - // 2.alter instance does not exist - sendResponseStatus = false; - AlertSendResponse.AlertSendResponseResult alertResult = new AlertSendResponse.AlertSendResponseResult(); - String message = String.format("Alert GroupId %s send error : not found alert instance", groupId); - alertResult.setSuccess(false); - alertResult.setMessage(message); - sendResponseResults.add(alertResult); - AlertSendResponse alertSendResponseData = - new AlertSendResponse(sendResponseStatus, sendResponseResults); - Message resMessage = alertSendResponseData.convert2Command(reqMessage.getOpaque()); - - Mockito.when(client.sendSync(Mockito.any(Host.class), Mockito.any(Message.class), Mockito.anyLong())) - .thenReturn(resMessage); - alertSendResponse = - alertClient.sendAlert(host, port, groupId, title, content, WarningType.FAILURE.getCode()); - Assertions.assertFalse(alertSendResponse.isSuccess()); - alertSendResponse.getResResults().forEach(result -> logger - .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); - - // 3.alter plugin does not exist - sendResponseStatus = false; - String pluginInstanceName = "alert-mail"; - message = String.format("Alert Plugin %s send error : return value is null", pluginInstanceName); - alertResult.setSuccess(false); - alertResult.setMessage(message); - alertSendResponseData = new AlertSendResponse(sendResponseStatus, sendResponseResults); - resMessage = alertSendResponseData.convert2Command(reqMessage.getOpaque()); - Mockito.when(client.sendSync(Mockito.any(), Mockito.any(), Mockito.anyLong())).thenReturn(resMessage); - alertSendResponse = - alertClient.sendAlert(host, port, groupId, title, content, WarningType.FAILURE.getCode()); - Assertions.assertFalse(alertSendResponse.isSuccess()); - alertSendResponse.getResResults().forEach(result -> logger - .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); - - // 4.alter result is null - sendResponseStatus = false; - message = String.format("Alert Plugin %s send error : return result value is null", pluginInstanceName); - alertResult.setSuccess(false); - alertResult.setMessage(message); - alertSendResponseData = new AlertSendResponse(sendResponseStatus, sendResponseResults); - resMessage = alertSendResponseData.convert2Command(reqMessage.getOpaque()); - Mockito.when(client.sendSync(Mockito.any(), Mockito.any(), Mockito.anyLong())).thenReturn(resMessage); - alertSendResponse = - alertClient.sendAlert(host, port, groupId, title, content, WarningType.FAILURE.getCode()); - Assertions.assertFalse(alertSendResponse.isSuccess()); - alertSendResponse.getResResults().forEach(result -> logger - .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); - - // 5.abnormal information inside the alert plug-in code - sendResponseStatus = false; - alertResult.setSuccess(false); - alertResult.setMessage("Abnormal information inside the alert plug-in code"); - alertSendResponseData = new AlertSendResponse(sendResponseStatus, sendResponseResults); - resMessage = alertSendResponseData.convert2Command(reqMessage.getOpaque()); - Mockito.when(client.sendSync(Mockito.any(), Mockito.any(), Mockito.anyLong())).thenReturn(resMessage); - alertSendResponse = - alertClient.sendAlert(host, port, groupId, title, content, WarningType.FAILURE.getCode()); - Assertions.assertFalse(alertSendResponse.isSuccess()); - alertSendResponse.getResResults().forEach(result -> logger - .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); - - // 6.alert plugin send success - sendResponseStatus = true; - message = String.format("Alert Plugin %s send success", pluginInstanceName); - alertResult.setSuccess(true); - alertResult.setMessage(message); - alertSendResponseData = new AlertSendResponse(sendResponseStatus, sendResponseResults); - resMessage = alertSendResponseData.convert2Command(reqMessage.getOpaque()); - Mockito.when(client.sendSync(Mockito.any(), Mockito.any(), Mockito.anyLong())).thenReturn(resMessage); - alertSendResponse = - alertClient.sendAlert(host, port, groupId, title, content, WarningType.FAILURE.getCode()); - Assertions.assertTrue(alertSendResponse.isSuccess()); - alertSendResponse.getResResults().forEach(result -> logger - .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); - - if (Objects.nonNull(alertClient) && alertClient.isRunning()) { - alertClient.close(); - } - - } - -} diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java deleted file mode 100644 index 6714ad2a29..0000000000 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.cache; - -import org.apache.dolphinscheduler.common.enums.CacheType; -import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.registry.api.RegistryClient; -import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.cache.CacheExpireRequest; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.service.cache.impl.CacheNotifyServiceImpl; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.netty.channel.Channel; - -/** - * tenant cache proxy test - */ -@ExtendWith(MockitoExtension.class) -public class CacheNotifyServiceTest { - - @InjectMocks - private CacheNotifyServiceImpl cacheNotifyService; - - @Mock - private RegistryClient registryClient; - - @Test - public void testNotifyMaster() { - User user1 = new User(); - user1.setId(100); - Message cacheExpireMessage = new CacheExpireRequest(CacheType.USER, "100").convert2Command(); - - NettyServerConfig serverConfig = new NettyServerConfig(); - - NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(new NettyRequestProcessor() { - - @Override - public void process(Channel channel, Message message) { - Assertions.assertEquals(cacheExpireMessage, message); - } - - @Override - public MessageType getCommandType() { - return MessageType.CACHE_EXPIRE; - } - }); - nettyRemotingServer.start(); - - List serverList = new ArrayList<>(); - Server server = new Server(); - server.setHost("127.0.0.1"); - server.setPort(serverConfig.getListenPort()); - serverList.add(server); - - Mockito.when(registryClient.getServerList(RegistryNodeType.MASTER)).thenReturn(serverList); - - cacheNotifyService.notifyMaster(cacheExpireMessage); - - nettyRemotingServer.close(); - } -} diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java deleted file mode 100644 index 794f8fb309..0000000000 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.service.log; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.log.ViewLogRequest; -import org.apache.dolphinscheduler.remote.processor.ViewWholeLogProcessor; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.netty.channel.Channel; - -@ExtendWith(MockitoExtension.class) -public class LoggerRequestProcessorTest { - - private MockedStatic mockedStaticLoggerUtils; - - @BeforeEach - public void setUp() { - mockedStaticLoggerUtils = Mockito.mockStatic(LogUtils.class); - } - - @AfterEach - public void after() { - mockedStaticLoggerUtils.close(); - } - - @Test - public void testProcessViewWholeLogRequest() { - System.setProperty("DOLPHINSCHEDULER_WORKER_HOME", System.getProperty("user.dir")); - Channel channel = Mockito.mock(Channel.class); - Mockito.when(channel.writeAndFlush(Mockito.any(Message.class))).thenReturn(null); - Mockito.when(LogUtils.readWholeFileContentFromLocal(Mockito.anyString())).thenReturn(""); - String userDir = System.getProperty("user.dir"); - ViewLogRequest logRequestCommand = new ViewLogRequest(userDir + "/log/path/a.log"); - - Message message = new Message(); - message.setType(MessageType.VIEW_WHOLE_LOG_REQUEST); - message.setBody(JSONUtils.toJsonByteArray(logRequestCommand)); - - ViewWholeLogProcessor loggerRequestProcessor = new ViewWholeLogProcessor(); - loggerRequestProcessor.process(channel, message); - } - - @Test - public void testProcessViewWholeLogRequestError() { - System.setProperty("DOLPHINSCHEDULER_WORKER_HOME", System.getProperty("user.dir")); - Channel channel = Mockito.mock(Channel.class); - Mockito.when(LogUtils.readWholeFileContentFromLocal(Mockito.anyString())).thenReturn(""); - String userDir = System.getProperty("user.dir"); - ViewLogRequest logRequestCommand = new ViewLogRequest(userDir + "/log/path/a"); - - Message message = new Message(); - message.setType(MessageType.VIEW_WHOLE_LOG_REQUEST); - message.setBody(JSONUtils.toJsonByteArray(logRequestCommand)); - - ViewWholeLogProcessor loggerRequestProcessor = new ViewWholeLogProcessor(); - loggerRequestProcessor.process(channel, message); - } - - @Test - public void testProcessViewWholeLogRequestErrorRelativePath() { - System.setProperty("DOLPHINSCHEDULER_WORKER_HOME", System.getProperty("user.dir")); - Channel channel = Mockito.mock(Channel.class); - Mockito.when(LogUtils.readWholeFileContentFromLocal(Mockito.anyString())).thenReturn(""); - String userDir = System.getProperty("user.dir"); - ViewLogRequest logRequestCommand = new ViewLogRequest(userDir + "/log/../../a.log"); - - Message message = new Message(); - message.setType(MessageType.VIEW_WHOLE_LOG_REQUEST); - message.setBody(JSONUtils.toJsonByteArray(logRequestCommand)); - - ViewWholeLogProcessor loggerRequestProcessor = new ViewWholeLogProcessor(); - loggerRequestProcessor.process(channel, message); - } - - @Test - public void testProcessViewWholeLogRequestErrorStartWith() { - System.setProperty("DOLPHINSCHEDULER_WORKER_HOME", System.getProperty("user.dir")); - Channel channel = Mockito.mock(Channel.class); - Mockito.when(LogUtils.readWholeFileContentFromLocal(Mockito.anyString())).thenReturn(""); - ViewLogRequest logRequestCommand = new ViewLogRequest("/log/a.log"); - - Message message = new Message(); - message.setType(MessageType.VIEW_WHOLE_LOG_REQUEST); - message.setBody(JSONUtils.toJsonByteArray(logRequestCommand)); - - ViewWholeLogProcessor loggerRequestProcessor = new ViewWholeLogProcessor(); - loggerRequestProcessor.process(channel, message); - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java index 2501e1e824..eef4c98e69 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java @@ -299,10 +299,8 @@ public abstract class AbstractCommandExecutor { ExecutorService getOutputLogService = ThreadUtils .newSingleDaemonScheduledExecutorService("ResolveOutputLog-thread-" + taskRequest.getTaskName()); getOutputLogService.submit(() -> { - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath()); - BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + try (BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath()); String line; while ((line = inReader.readLine()) != null) { if (line.startsWith("${setValue(") || line.startsWith("#{setValue(")) { @@ -316,6 +314,8 @@ public abstract class AbstractCommandExecutor { } catch (Exception e) { logger.error("Parse var pool error", e); processLogOutputIsSuccess = true; + } finally { + LogUtils.removeTaskInstanceLogFullPathMDC(); } }); @@ -324,9 +324,8 @@ public abstract class AbstractCommandExecutor { ExecutorService parseProcessOutputExecutorService = ThreadUtils .newSingleDaemonScheduledExecutorService("TaskInstanceLogOutput-thread-" + taskRequest.getTaskName()); taskOutputFuture = parseProcessOutputExecutorService.submit(() -> { - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath());) { + try { + LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath()); while (logBuffer.size() > 1 || !processLogOutputIsSuccess || !podLogOutputIsFinished) { if (logBuffer.size() > 1) { logHandler.accept(logBuffer); @@ -338,6 +337,8 @@ public abstract class AbstractCommandExecutor { } } catch (Exception e) { logger.error("Output task log error", e); + } finally { + LogUtils.removeTaskInstanceLogFullPathMDC(); } }); parseProcessOutputExecutorService.shutdown(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java index d799978763..ae78c66459 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java @@ -194,9 +194,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { @Override public void eventReceived(Action action, Job job) { - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath())) { + try { + LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath()); log.info("event received : job:{} action:{}", job.getMetadata().getName(), action); if (action == Action.DELETED) { log.error("[K8sJobExecutor-{}] fail in k8s", job.getMetadata().getName()); @@ -211,6 +210,8 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { setTaskStatus(jobStatus, taskInstanceId, taskResponse); countDownLatch.countDown(); } + } finally { + LogUtils.removeTaskInstanceLogFullPathMDC(); } } @@ -249,10 +250,9 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { String containerName = String.format("%s-%s", taskName, taskInstanceId); podLogOutputFuture = collectPodLogExecutorService.submit(() -> { try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath()); LogWatch watcher = ProcessUtils.getPodLogWatcher(taskRequest.getK8sTaskExecutionContext(), taskRequest.getTaskAppId(), containerName)) { + LogUtils.setTaskInstanceLogFullPathMDC(taskRequest.getLogPath()); String line; try (BufferedReader reader = new BufferedReader(new InputStreamReader(watcher.getOutput()))) { while ((line = reader.readLine()) != null) { @@ -262,6 +262,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor { } catch (Exception e) { throw new RuntimeException(e); } finally { + LogUtils.removeTaskInstanceLogFullPathMDC(); podLogOutputIsFinished = true; } }); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/LogUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/LogUtils.java index ceea6f8302..61d901d80c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/LogUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/LogUtils.java @@ -188,30 +188,30 @@ public class LogUtils { return MDC.get(TASK_INSTANCE_LOG_FULL_PATH_MDC_KEY); } - public static MDCAutoClosableContext setTaskInstanceLogFullPathMDC(String taskInstanceLogFullPath) { + public static void setTaskInstanceLogFullPathMDC(String taskInstanceLogFullPath) { + if (taskInstanceLogFullPath == null) { + log.warn("taskInstanceLogFullPath is null"); + return; + } MDC.put(TASK_INSTANCE_LOG_FULL_PATH_MDC_KEY, taskInstanceLogFullPath); - return new MDCAutoClosableContext(LogUtils::removeTaskInstanceLogFullPathMDC); } public static void removeTaskInstanceLogFullPathMDC() { MDC.remove(TASK_INSTANCE_LOG_FULL_PATH_MDC_KEY); } - public static MDCAutoClosableContext setWorkflowAndTaskInstanceIDMDC(Integer workflowInstanceId, - Integer taskInstanceId) { + public static void setWorkflowAndTaskInstanceIDMDC(Integer workflowInstanceId, + Integer taskInstanceId) { MDC.put(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY, String.valueOf(workflowInstanceId)); MDC.put(Constants.TASK_INSTANCE_ID_MDC_KEY, String.valueOf(taskInstanceId)); - return new MDCAutoClosableContext(LogUtils::removeWorkflowAndTaskInstanceIdMDC); } - public static MDCAutoClosableContext setWorkflowInstanceIdMDC(Integer workflowInstanceId) { + public static void setWorkflowInstanceIdMDC(Integer workflowInstanceId) { MDC.put(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY, String.valueOf(workflowInstanceId)); - return new MDCAutoClosableContext(LogUtils::removeWorkflowInstanceIdMDC); } - public static MDCAutoClosableContext setTaskInstanceIdMDC(Integer taskInstanceId) { + public static void setTaskInstanceIdMDC(Integer taskInstanceId) { MDC.put(Constants.TASK_INSTANCE_ID_MDC_KEY, String.valueOf(taskInstanceId)); - return new MDCAutoClosableContext(LogUtils::removeTaskInstanceIdMDC); } public static void removeWorkflowAndTaskInstanceIdMDC() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java new file mode 100644 index 0000000000..606058d40f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/TaskUtils.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.plugin.task.api.utils; + +import java.util.Set; + +import lombok.experimental.UtilityClass; + +import com.google.common.collect.Sets; + +@UtilityClass +public class TaskUtils { + + private final String blockingLogicTask = "BLOCKING"; + private final String conditionLogicTask = "CONDITIONS"; + + private final String dependentLogicTask = "DEPENDENT"; + private final String subWorkflowLogicTask = "SUB_PROCESS"; + private final String switchLogicTask = "SWITCH"; + private final String dynamicLogicTask = "DYNAMIC"; + + // todo: Add to SPI + private final Set MASTER_TASK_TYPES = Sets.newHashSet( + blockingLogicTask, + conditionLogicTask, + dependentLogicTask, + subWorkflowLogicTask, + switchLogicTask, + dynamicLogicTask); + + // todo: add to task plugin spi + public boolean isLogicTask(String taskType) { + return MASTER_TASK_TYPES.contains(taskType); + } + +} diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index 60cda1a12f..700ecb90d6 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -46,10 +46,7 @@ org.apache.dolphinscheduler dolphinscheduler-common - - org.apache.dolphinscheduler - dolphinscheduler-remote - + org.apache.dolphinscheduler dolphinscheduler-meter @@ -69,6 +66,21 @@ dolphinscheduler-storage-all + + org.apache.dolphinscheduler + dolphinscheduler-extract-alert + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-master + + + + org.apache.dolphinscheduler + dolphinscheduler-extract-worker + + org.springframework.boot spring-boot-starter diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 5e5d6219a4..cb301f9468 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcServer; +import org.apache.dolphinscheduler.server.worker.runner.GlobalTaskInstanceDispatchQueueLooper; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.commons.collections4.CollectionUtils; @@ -71,15 +71,15 @@ public class WorkerServer implements IStoppable { @Autowired private WorkerRpcServer workerRpcServer; - @Autowired - private WorkerRpcClient workerRpcClient; - @Autowired private MessageRetryRunner messageRetryRunner; @Autowired private WorkerConfig workerConfig; + @Autowired + private GlobalTaskInstanceDispatchQueueLooper globalTaskInstanceDispatchQueueLooper; + /** * worker server startup, not use web service * @@ -93,7 +93,6 @@ public class WorkerServer implements IStoppable { @PostConstruct public void run() { this.workerRpcServer.start(); - this.workerRpcClient.start(); this.taskPluginManager.loadPlugin(); this.workerRegistryClient.setRegistryStoppable(this); @@ -102,6 +101,7 @@ public class WorkerServer implements IStoppable { this.workerManagerThread.start(); this.messageRetryRunner.start(); + this.globalTaskInstanceDispatchQueueLooper.start(); /* * registry hooks, which are called before the process exits @@ -147,13 +147,14 @@ public class WorkerServer implements IStoppable { int killNumber = 0; for (TaskExecutionContext taskRequest : taskRequests) { // kill task when it's not finished yet - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setWorkflowAndTaskInstanceIDMDC(taskRequest.getProcessInstanceId(), - taskRequest.getTaskInstanceId())) { + try { + LogUtils.setWorkflowAndTaskInstanceIDMDC(taskRequest.getProcessInstanceId(), + taskRequest.getTaskInstanceId()); if (ProcessUtils.kill(taskRequest)) { killNumber++; } + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); } } log.info("Worker after kill all cache task, task size: {}, killed number: {}", taskRequests.size(), diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 981a15ecb5..4dd67613fe 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -21,8 +21,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.REGISTRY_DO import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.commons.lang3.StringUtils; @@ -54,9 +52,6 @@ public class WorkerConfig implements Validator { private double reservedMemory = 0.1; private ConnectStrategyProperties registryDisconnectStrategy = new ConnectStrategyProperties(); - private NettyClientConfig workerRpcClientConfig = new NettyClientConfig(); - private NettyServerConfig workerRpcServerConfig = new NettyServerConfig(); - /** * This field doesn't need to set at config file, it will be calculated by workerIp:listenPort */ diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java index 44befbe31d..b5a388312b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java @@ -20,9 +20,8 @@ package org.apache.dolphinscheduler.server.worker.message; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; import org.apache.commons.collections4.MapUtils; @@ -57,9 +56,10 @@ public class MessageRetryRunner extends BaseDaemonThread { @Lazy @Autowired - private List messageSenders; + private List messageSenders; - private final Map> messageSenderMap = new HashMap<>(); + private final Map> messageSenderMap = + new HashMap<>(); private final Map> needToRetryMessages = new ConcurrentHashMap<>(); @@ -74,15 +74,17 @@ public class MessageRetryRunner extends BaseDaemonThread { log.info("Message retry runner started"); } - public void addRetryMessage(int taskInstanceId, @NonNull MessageType messageType, BaseMessage baseMessage) { + public void addRetryMessage(int taskInstanceId, @NonNull ITaskInstanceExecutionEvent iTaskInstanceExecutionEvent) { needToRetryMessages.computeIfAbsent(taskInstanceId, k -> Collections.synchronizedList(new ArrayList<>())) - .add(TaskInstanceMessage.of(taskInstanceId, messageType, baseMessage)); + .add(TaskInstanceMessage.of(taskInstanceId, iTaskInstanceExecutionEvent.getEventType(), + iTaskInstanceExecutionEvent)); } - public void removeRetryMessage(int taskInstanceId, @NonNull MessageType messageType) { + public void removeRetryMessage(int taskInstanceId, + @NonNull ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType eventType) { List taskInstanceMessages = needToRetryMessages.get(taskInstanceId); if (taskInstanceMessages != null) { - taskInstanceMessages.remove(TaskInstanceMessage.of(taskInstanceId, messageType, null)); + taskInstanceMessages.remove(TaskInstanceMessage.of(taskInstanceId, eventType, null)); } } @@ -94,7 +96,7 @@ public class MessageRetryRunner extends BaseDaemonThread { List taskInstanceMessages = this.needToRetryMessages.get(taskInstanceId); if (taskInstanceMessages != null) { taskInstanceMessages.forEach(taskInstanceMessage -> { - taskInstanceMessage.getMessage().setMessageReceiverAddress(messageReceiverHost); + taskInstanceMessage.getEvent().setHost(messageReceiverHost); }); } } @@ -120,13 +122,14 @@ public class MessageRetryRunner extends BaseDaemonThread { LogUtils.setTaskInstanceIdMDC(taskInstanceId); try { for (TaskInstanceMessage taskInstanceMessage : taskInstanceMessages) { - MessageType messageType = taskInstanceMessage.getMessageType(); - BaseMessage message = taskInstanceMessage.getMessage(); - if (now - message.getMessageSendTime() > MESSAGE_RETRY_WINDOW) { - log.info("Begin retry send message to master, message: {}", message); - message.setMessageSendTime(now); - messageSenderMap.get(messageType).sendMessage(message); - log.info("Success send message to master, message: {}", message); + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType eventType = + taskInstanceMessage.getEventType(); + ITaskInstanceExecutionEvent event = taskInstanceMessage.getEvent(); + if (now - event.getEventSendTime() > MESSAGE_RETRY_WINDOW) { + log.info("Begin retry send message to master, event: {}", event); + event.setEventSendTime(now); + messageSenderMap.get(eventType).sendEvent(event); + log.info("Success send message to master, event: {}", event); } } } catch (Exception e) { @@ -157,14 +160,16 @@ public class MessageRetryRunner extends BaseDaemonThread { public static class TaskInstanceMessage { private long taskInstanceId; - private MessageType messageType; - private BaseMessage message; + private ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType eventType; + private ITaskInstanceExecutionEvent event; - public static TaskInstanceMessage of(long taskInstanceId, MessageType messageType, BaseMessage message) { + public static TaskInstanceMessage of(long taskInstanceId, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType eventType, + ITaskInstanceExecutionEvent event) { TaskInstanceMessage taskInstanceMessage = new TaskInstanceMessage(); taskInstanceMessage.setTaskInstanceId(taskInstanceId); - taskInstanceMessage.setMessageType(messageType); - taskInstanceMessage.setMessage(message); + taskInstanceMessage.setEventType(eventType); + taskInstanceMessage.setEvent(event); return taskInstanceMessage; } @@ -177,12 +182,12 @@ public class MessageRetryRunner extends BaseDaemonThread { return false; } TaskInstanceMessage that = (TaskInstanceMessage) o; - return taskInstanceId == that.taskInstanceId && messageType == that.messageType; + return taskInstanceId == that.taskInstanceId && eventType == that.eventType; } @Override public int hashCode() { - return Objects.hashCode(taskInstanceId, messageType); + return Objects.hashCode(taskInstanceId, eventType); } } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskInstanceExecutionEventSender.java similarity index 70% rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskInstanceExecutionEventSender.java index 79bc7eec6c..a77fa3b668 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskInstanceExecutionEventSender.java @@ -17,27 +17,24 @@ package org.apache.dolphinscheduler.server.worker.message; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -public interface MessageSender { +public interface TaskInstanceExecutionEventSender { /** * Send the message * - * @throws RemotingException Cannot connect to the target host. */ - void sendMessage(T message) throws RemotingException; + void sendEvent(T message); /** * Build the message from task context and message received address. */ - T buildMessage(TaskExecutionContext taskExecutionContext); + T buildEvent(TaskExecutionContext taskExecutionContext); /** * The message type can be sent by this sender. */ - MessageType getMessageType(); + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType getMessageType(); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskInstanceExecutionFinishEventSender.java similarity index 60% rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskInstanceExecutionFinishEventSender.java index 2aed7b69cc..27a9de73ae 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskInstanceExecutionFinishEventSender.java @@ -17,37 +17,31 @@ package org.apache.dolphinscheduler.server.worker.message; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; +import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessage; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component -public class TaskExecuteResultMessageSender implements MessageSender { - - @Autowired - private WorkerConfig workerConfig; - - @Autowired - private WorkerRpcClient workerRpcClient; +public class TaskInstanceExecutionFinishEventSender + implements + TaskInstanceExecutionEventSender { @Override - public void sendMessage(TaskExecuteResultMessage message) throws RemotingException { - workerRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command()); + public void sendEvent(TaskInstanceExecutionFinishEvent taskInstanceExecutionFinishEvent) { + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstanceExecutionFinishEvent.getHost(), + ITaskInstanceExecutionEventListener.class); + iTaskInstanceExecutionEventListener.onTaskInstanceExecutionFinish(taskInstanceExecutionFinishEvent); } @Override - public TaskExecuteResultMessage buildMessage(TaskExecutionContext taskExecutionContext) { - TaskExecuteResultMessage taskExecuteResultMessage = - new TaskExecuteResultMessage(workerConfig.getWorkerAddress(), - taskExecutionContext.getWorkflowInstanceHost(), - System.currentTimeMillis()); + public TaskInstanceExecutionFinishEvent buildEvent(TaskExecutionContext taskExecutionContext) { + TaskInstanceExecutionFinishEvent taskExecuteResultMessage = new TaskInstanceExecutionFinishEvent(); taskExecuteResultMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskExecuteResultMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskExecuteResultMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); @@ -64,7 +58,7 @@ public class TaskExecuteResultMessageSender implements MessageSender { - - @Autowired - private WorkerRpcClient workerRpcClient; - - @Autowired - private WorkerConfig workerConfig; +public class TaskInstanceExecutionInfoUpdateEventSender + implements + TaskInstanceExecutionEventSender { @Override - public void sendMessage(TaskUpdateRuntimeMessage message) throws RemotingException { - workerRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command()); + public void sendEvent(TaskInstanceExecutionInfoEvent taskInstanceExecutionInfoEvent) { + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstanceExecutionInfoEvent.getHost(), + ITaskInstanceExecutionEventListener.class); + iTaskInstanceExecutionEventListener.onTaskInstanceExecutionInfoUpdate(taskInstanceExecutionInfoEvent); } @Override - public TaskUpdateRuntimeMessage buildMessage(@NonNull TaskExecutionContext taskExecutionContext) { - TaskUpdateRuntimeMessage taskUpdatePidRequest = - new TaskUpdateRuntimeMessage(workerConfig.getWorkerAddress(), - taskExecutionContext.getWorkflowInstanceHost(), - System.currentTimeMillis()); + public TaskInstanceExecutionInfoEvent buildEvent(@NonNull TaskExecutionContext taskExecutionContext) { + TaskInstanceExecutionInfoEvent taskUpdatePidRequest = new TaskInstanceExecutionInfoEvent(); taskUpdatePidRequest.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskUpdatePidRequest.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskUpdatePidRequest.setHost(taskExecutionContext.getHost()); @@ -58,7 +52,7 @@ public class WorkerTaskUpdateRuntimeMessageSender implements MessageSender { - - @Autowired - private WorkerRpcClient workerRpcClient; - - @Autowired - private WorkerConfig workerConfig; +public class TaskInstanceExecutionRunningEventSender + implements + TaskInstanceExecutionEventSender { @Override - public void sendMessage(TaskExecuteRunningMessage message) throws RemotingException { - workerRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command()); + public void sendEvent(TaskInstanceExecutionRunningEvent taskInstanceExecutionRunningEvent) { + ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener = + SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(taskInstanceExecutionRunningEvent.getHost(), + ITaskInstanceExecutionEventListener.class); + iTaskInstanceExecutionEventListener.onTaskInstanceExecutionRunning(taskInstanceExecutionRunningEvent); } @Override - public TaskExecuteRunningMessage buildMessage(@NonNull TaskExecutionContext taskExecutionContext) { - TaskExecuteRunningMessage taskExecuteRunningMessage = - new TaskExecuteRunningMessage(workerConfig.getWorkerAddress(), - taskExecutionContext.getWorkflowInstanceHost(), - System.currentTimeMillis()); + public TaskInstanceExecutionRunningEvent buildEvent(@NonNull TaskExecutionContext taskExecutionContext) { + TaskInstanceExecutionRunningEvent taskExecuteRunningMessage = new TaskInstanceExecutionRunningEvent(); taskExecuteRunningMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskExecuteRunningMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskExecuteRunningMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus()); @@ -62,7 +56,7 @@ public class TaskExecuteRunningMessageSender implements MessageSender 0) { - log.info("Current taskInstance is choose delay execution, delay time: {}s", remainTime); - taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.DELAY_EXECUTION); - workerMessageSender.sendMessage(taskExecutionContext, MessageType.TASK_EXECUTE_RESULT_MESSAGE); - } - - WorkerDelayTaskExecuteRunnable workerTaskExecuteRunnable = WorkerTaskExecuteRunnableFactoryBuilder - .createWorkerDelayTaskExecuteRunnableFactory( - taskExecutionContext, - workerConfig, - workerMessageSender, - workerRpcClient, - taskPluginManager, - storageOperate, - workerRegistryClient) - .createWorkerTaskExecuteRunnable(); - if (!workerManager.offer(workerTaskExecuteRunnable)) { - log.error("submit task: {} to wait queue error, current queue size: {} is full", - taskExecutionContext.getTaskName(), workerManager.getWaitSubmitQueueSize()); - sendDispatchRejectResult(channel, message, taskExecutionContext); - } else { - sendDispatchSuccessResult(channel, message, taskExecutionContext); - log.info("Submit task: {} to wait queue success", taskExecutionContext.getTaskName()); - } - } - } - - private void sendDispatchSuccessResult(Channel channel, Message dispatchRequest, - TaskExecutionContext taskExecutionContext) { - TaskDispatchResponse taskDispatchResponse = - TaskDispatchResponse.success(taskExecutionContext.getTaskInstanceId()); - channel.writeAndFlush(taskDispatchResponse.convert2Command(dispatchRequest.getOpaque())); - } - - private void sendDispatchRejectResult(Channel channel, Message dispatchRequest, - TaskExecutionContext taskExecutionContext) { - TaskDispatchResponse taskDispatchResponse = - TaskDispatchResponse.failed(taskExecutionContext.getTaskInstanceId(), "Task dispatch queue is full"); - channel.writeAndFlush(taskDispatchResponse.convert2Command(dispatchRequest.getOpaque())); - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_DISPATCH_REQUEST; - } - -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteResultAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteResultAckProcessor.java deleted file mode 100644 index 92816624b1..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteResultAckProcessor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteResultMessageAck; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; -import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute running ack, from master to worker - */ -@Slf4j -@Component -public class WorkerTaskExecuteResultAckProcessor implements WorkerRpcProcessor { - - @Autowired - private MessageRetryRunner messageRetryRunner; - - @Override - public void process(Channel channel, Message message) { - TaskExecuteResultMessageAck taskExecuteAckMessage = - JSONUtils.parseObject(message.getBody(), TaskExecuteResultMessageAck.class); - - if (taskExecuteAckMessage == null) { - log.error("task execute response ack command is null"); - return; - } - - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceIdMDC(taskExecuteAckMessage.getTaskInstanceId())) { - log.info("Receive task execute response ack command : {}", taskExecuteAckMessage); - if (taskExecuteAckMessage.isSuccess()) { - messageRetryRunner.removeRetryMessage(taskExecuteAckMessage.getTaskInstanceId(), - MessageType.TASK_EXECUTE_RESULT_MESSAGE); - log.debug("remove REMOTE_CHANNELS, task instance id:{}", taskExecuteAckMessage.getTaskInstanceId()); - } else { - // master handle worker response error, will still retry - log.error("Receive task execute result ack message, the message status is not success, message: {}", - taskExecuteAckMessage); - } - } - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RESULT_MESSAGE_ACK; - } - -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteRunningAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteRunningAckProcessor.java deleted file mode 100644 index 216ecdc83c..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskExecuteRunningAckProcessor.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskExecuteRunningMessageAck; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; -import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute running ack processor - */ -@Component -@Slf4j -public class WorkerTaskExecuteRunningAckProcessor implements WorkerRpcProcessor { - - @Autowired - private MessageRetryRunner messageRetryRunner; - - @Override - public void process(Channel channel, Message message) { - TaskExecuteRunningMessageAck runningAckCommand = JSONUtils.parseObject(message.getBody(), - TaskExecuteRunningMessageAck.class); - if (runningAckCommand == null) { - log.error("task execute running ack command is null"); - return; - } - try { - LogUtils.setTaskInstanceIdMDC(runningAckCommand.getTaskInstanceId()); - log.info("task execute running ack command : {}", runningAckCommand); - - if (runningAckCommand.isSuccess()) { - messageRetryRunner.removeRetryMessage(runningAckCommand.getTaskInstanceId(), - MessageType.TASK_EXECUTE_RUNNING_MESSAGE); - } - } finally { - LogUtils.removeTaskInstanceIdMDC(); - } - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_EXECUTE_RUNNING_MESSAGE_ACK; - } - -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskPauseProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskPauseProcessor.java deleted file mode 100644 index a3b43b21b1..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskPauseProcessor.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskPauseRequest; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -@Slf4j -@Component -public class WorkerTaskPauseProcessor implements WorkerRpcProcessor { - - @Override - public void process(Channel channel, Message message) { - TaskPauseRequest taskPauseRequest = JSONUtils.parseObject(message.getBody(), TaskPauseRequest.class); - log.info("Receive TaskPauseRequest: {}, worker doesn't support pause", taskPauseRequest.getTaskInstanceId()); - } - - @Override - public MessageType getCommandType() { - return MessageType.PAUSE_TASK_INSTANCE; - } -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskSavePointProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskSavePointProcessor.java deleted file mode 100644 index 55fbfbfe28..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskSavePointProcessor.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; -import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTask; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskSavePointRequest; -import org.apache.dolphinscheduler.remote.command.task.TaskSavePointResponse; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; -import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; -import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; - -/** - * task save point processor - */ -@Component -@Slf4j -public class WorkerTaskSavePointProcessor implements WorkerRpcProcessor { - - /** - * task execute manager - */ - @Autowired - private WorkerManagerThread workerManager; - - /** - * task save point process - * - * @param channel channel channel - * @param message command command - */ - @Override - public void process(Channel channel, Message message) { - TaskSavePointRequest taskSavePointRequest = - JSONUtils.parseObject(message.getBody(), TaskSavePointRequest.class); - if (taskSavePointRequest == null) { - log.error("task savepoint request command is null"); - return; - } - log.info("Receive task savepoint command : {}", taskSavePointRequest); - - int taskInstanceId = taskSavePointRequest.getTaskInstanceId(); - TaskExecutionContext taskExecutionContext = - TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); - if (taskExecutionContext == null) { - log.error("taskRequest cache is null, taskInstanceId: {}", - taskSavePointRequest.getTaskInstanceId()); - return; - } - - try { - LogUtils.setTaskInstanceIdMDC(taskInstanceId); - doSavePoint(taskInstanceId); - - sendTaskSavePointResponseCommand(channel, taskExecutionContext); - } finally { - LogUtils.removeTaskInstanceIdMDC(); - } - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_SAVEPOINT_REQUEST; - } - - private void sendTaskSavePointResponseCommand(Channel channel, TaskExecutionContext taskExecutionContext) { - TaskSavePointResponse taskSavePointResponse = new TaskSavePointResponse(); - taskSavePointResponse.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - channel.writeAndFlush(taskSavePointResponse.convert2Command()).addListener(new ChannelFutureListener() { - - @Override - public void operationComplete(ChannelFuture future) throws Exception { - if (!future.isSuccess()) { - log.error("Submit kill response to master error, kill command: {}", - taskSavePointResponse); - } else - log.info("Submit kill response to master success, kill command: {}", - taskSavePointResponse); - } - }); - } - - protected void doSavePoint(int taskInstanceId) { - WorkerTaskExecuteRunnable workerTaskExecuteRunnable = workerManager.getTaskExecuteThread(taskInstanceId); - if (workerTaskExecuteRunnable == null) { - log.warn("taskExecuteThread not found, taskInstanceId:{}", taskInstanceId); - return; - } - AbstractTask task = workerTaskExecuteRunnable.getTask(); - if (task == null) { - log.warn("task not found, taskInstanceId:{}", taskInstanceId); - return; - } - if (!(task instanceof StreamTask)) { - log.warn("task is not stream task"); - return; - } - try { - ((StreamTask) task).savePoint(); - } catch (Exception e) { - log.error("task save point error", e); - } - } -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskUpdatePidAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskUpdatePidAckProcessor.java deleted file mode 100644 index add954d2ec..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskUpdatePidAckProcessor.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskUpdateRuntimeAckMessage; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; -import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; - -import javax.annotation.Resource; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; - -/** - * task execute running ack processor - */ -@Component -@Slf4j -public class WorkerTaskUpdatePidAckProcessor implements WorkerRpcProcessor { - - @Resource - private MessageRetryRunner messageRetryRunner; - - @Override - public void process(Channel channel, Message message) { - - TaskUpdateRuntimeAckMessage updatePidAckCommand = JSONUtils.parseObject(message.getBody(), - TaskUpdateRuntimeAckMessage.class); - if (updatePidAckCommand == null) { - log.error("task execute update pid ack command is null"); - return; - } - try ( - LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceIdMDC(updatePidAckCommand.getTaskInstanceId());) { - LogUtils.setTaskInstanceIdMDC(updatePidAckCommand.getTaskInstanceId()); - log.info("task execute update pid ack command : {}", updatePidAckCommand); - - if (updatePidAckCommand.isSuccess()) { - messageRetryRunner.removeRetryMessage(updatePidAckCommand.getTaskInstanceId(), - MessageType.TASK_UPDATE_RUNTIME_MESSAGE); - } - } - } - - @Override - public MessageType getCommandType() { - return MessageType.TASK_UPDATE_RUNTIME_MESSAGE_ACK; - } - -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkflowHostChangeProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkflowHostChangeProcessor.java deleted file mode 100644 index 4d4d87020a..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkflowHostChangeProcessor.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; -import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.WorkflowHostChangeRequest; -import org.apache.dolphinscheduler.remote.command.task.WorkflowHostChangeResponse; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; -import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelFutureListener; - -/** - * update process host - * this used when master failover - */ -@Component -public class WorkflowHostChangeProcessor implements WorkerRpcProcessor { - - private final Logger logger = LoggerFactory.getLogger(WorkflowHostChangeProcessor.class); - - @Autowired - private MessageRetryRunner messageRetryRunner; - - @Override - public void process(Channel channel, Message message) { - WorkflowHostChangeRequest workflowHostChangeRequest = - JSONUtils.parseObject(message.getBody(), WorkflowHostChangeRequest.class); - if (workflowHostChangeRequest == null) { - logger.error("host update command is null"); - return; - } - logger.info("Received workflow host change command : {}", workflowHostChangeRequest); - try ( - final LogUtils.MDCAutoClosableContext mdcAutoClosableContext = - LogUtils.setTaskInstanceIdMDC(workflowHostChangeRequest.getTaskInstanceId())) { - WorkflowHostChangeResponse workflowHostChangeResponse; - TaskExecutionContext taskExecutionContext = - TaskExecutionContextCacheManager.getByTaskInstanceId(workflowHostChangeRequest.getTaskInstanceId()); - if (taskExecutionContext != null) { - taskExecutionContext.setWorkflowInstanceHost(workflowHostChangeRequest.getWorkflowHost()); - messageRetryRunner.updateMessageHost(workflowHostChangeRequest.getTaskInstanceId(), - workflowHostChangeRequest.getWorkflowHost()); - workflowHostChangeResponse = WorkflowHostChangeResponse.success(); - logger.info("Success update workflow host, taskInstanceId : {}, workflowHost: {}", - workflowHostChangeRequest.getTaskInstanceId(), workflowHostChangeRequest.getWorkflowHost()); - } else { - workflowHostChangeResponse = WorkflowHostChangeResponse.failed(); - logger.error("Cannot find the taskExecutionContext, taskInstanceId : {}", - workflowHostChangeRequest.getTaskInstanceId()); - } - channel.writeAndFlush(workflowHostChangeResponse.convert2Command(message.getOpaque())).addListener( - (ChannelFutureListener) channelFuture -> { - if (!channelFuture.isSuccess()) { - logger.error("send host update response failed"); - } - }); - } - } - - @Override - public MessageType getCommandType() { - return MessageType.WORKFLOW_HOST_CHANGE_REQUEST; - } - -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java index 4a049f7d90..ab60de9a77 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java @@ -25,10 +25,10 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.registry.api.RegistryClient; import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.task.WorkerHeartBeatTask; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java index bc3563d5a1..2ab6ed8ecd 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java @@ -24,8 +24,8 @@ import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.StrategyType; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcServer; +import org.apache.dolphinscheduler.server.worker.runner.GlobalTaskInstanceDispatchQueue; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import java.time.Duration; @@ -50,15 +50,15 @@ public class WorkerWaitingStrategy implements WorkerConnectStrategy { @Autowired private WorkerRpcServer workerRpcServer; - @Autowired - private WorkerRpcClient workerRpcClient; - @Autowired private MessageRetryRunner messageRetryRunner; @Autowired private WorkerManagerThread workerManagerThread; + @Autowired + private GlobalTaskInstanceDispatchQueue globalTaskInstanceDispatchQueue; + @Override public void disconnect() { try { @@ -120,9 +120,8 @@ public class WorkerWaitingStrategy implements WorkerConnectStrategy { // close the worker resource, if close failed should stop the worker server workerRpcServer.close(); log.warn("Worker server close the RPC server due to lost connection from registry"); - workerRpcClient.close(); - log.warn("Worker server close the RPC client due to lost connection from registry"); workerManagerThread.clearTask(); + globalTaskInstanceDispatchQueue.clearTask(); log.warn("Worker server clear the tasks due to lost connection from registry"); messageRetryRunner.clearMessage(); log.warn("Worker server clear the retry message due to lost connection from registry"); @@ -133,7 +132,5 @@ public class WorkerWaitingStrategy implements WorkerConnectStrategy { // reopen the resource, if reopen failed should stop the worker server workerRpcServer.start(); log.warn("Worker server restart PRC server due to reconnect to registry"); - workerRpcClient.start(); - log.warn("Worker server restart PRC client due to reconnect to registry"); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/StreamingTaskInstanceOperatorImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/StreamingTaskInstanceOperatorImpl.java new file mode 100644 index 0000000000..3466d52c28 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/StreamingTaskInstanceOperatorImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.rpc; + +import org.apache.dolphinscheduler.extract.worker.IStreamingTaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointResponse; +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; +import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTask; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; +import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class StreamingTaskInstanceOperatorImpl implements IStreamingTaskInstanceOperator { + + @Autowired + private WorkerManagerThread workerManager; + + @Override + public TaskInstanceTriggerSavepointResponse triggerSavepoint(TaskInstanceTriggerSavepointRequest taskInstanceTriggerSavepointRequest) { + log.info("Receive triggerSavepoint request: {}", taskInstanceTriggerSavepointRequest); + + try { + int taskInstanceId = taskInstanceTriggerSavepointRequest.getTaskInstanceId(); + LogUtils.setTaskInstanceIdMDC(taskInstanceId); + TaskExecutionContext taskExecutionContext = + TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); + if (taskExecutionContext == null) { + log.error("Cannot find TaskExecutionContext for taskInstance: {}", taskInstanceId); + return TaskInstanceTriggerSavepointResponse.fail("Cannot find TaskExecutionContext"); + } + WorkerTaskExecuteRunnable workerTaskExecuteRunnable = workerManager.getTaskExecuteThread(taskInstanceId); + if (workerTaskExecuteRunnable == null) { + log.error("Cannot find WorkerTaskExecuteRunnable for taskInstance: {}", taskInstanceId); + return TaskInstanceTriggerSavepointResponse.fail("Cannot find WorkerTaskExecuteRunnable"); + } + AbstractTask task = workerTaskExecuteRunnable.getTask(); + if (task == null) { + log.error("Cannot find StreamTask for taskInstance:{}", taskInstanceId); + return TaskInstanceTriggerSavepointResponse.fail("Cannot find StreamTask"); + } + if (!(task instanceof StreamTask)) { + log.warn("The taskInstance: {} is not StreamTask", taskInstanceId); + return TaskInstanceTriggerSavepointResponse.fail("The taskInstance is not StreamTask"); + } + try { + ((StreamTask) task).savePoint(); + } catch (Exception e) { + log.error("StreamTask: {} call savePoint error", taskInstanceId, e); + return TaskInstanceTriggerSavepointResponse.fail("StreamTask call savePoint error: " + e.getMessage()); + } + return TaskInstanceTriggerSavepointResponse.success(); + } finally { + LogUtils.removeTaskInstanceIdMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceExecutionEventAckListenerImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceExecutionEventAckListenerImpl.java new file mode 100644 index 0000000000..af9b7c16ec --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceExecutionEventAckListenerImpl.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.rpc; + +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionFinishEventAck; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionInfoEventAck; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck; +import org.apache.dolphinscheduler.server.worker.runner.listener.TaskInstanceExecutionEventAckListenFunctionManager; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionEventAckListenerImpl implements ITaskInstanceExecutionEventAckListener { + + @Autowired + private TaskInstanceExecutionEventAckListenFunctionManager taskInstanceExecutionEventAckListenFunctionManager; + + @Override + public void handleTaskInstanceExecutionRunningEventAck(TaskInstanceExecutionRunningEventAck taskInstanceExecutionRunningEventAck) { + taskInstanceExecutionEventAckListenFunctionManager.getTaskInstanceExecutionRunningEventAckListenFunction() + .handleTaskInstanceExecutionEventAck(taskInstanceExecutionRunningEventAck); + } + + @Override + public void handleTaskInstanceExecutionFinishEventAck(TaskInstanceExecutionFinishEventAck taskInstanceExecutionFinishEventAck) { + taskInstanceExecutionEventAckListenFunctionManager.getTaskInstanceExecutionFinishEventAckListenFunction() + .handleTaskInstanceExecutionEventAck(taskInstanceExecutionFinishEventAck); + } + + @Override + public void handleTaskInstanceExecutionInfoEventAck(TaskInstanceExecutionInfoEventAck taskInstanceExecutionInfoEventAck) { + taskInstanceExecutionEventAckListenFunctionManager.getTaskInstanceExecutionInfoEventAckListenFunction() + .handleTaskInstanceExecutionEventAck(taskInstanceExecutionInfoEventAck); + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceOperatorImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceOperatorImpl.java new file mode 100644 index 0000000000..4ca52a3e75 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/TaskInstanceOperatorImpl.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.rpc; + +import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; +import org.apache.dolphinscheduler.server.worker.runner.operator.TaskInstanceOperationFunctionManager; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TaskInstanceOperatorImpl implements ITaskInstanceOperator { + + @Autowired + private TaskInstanceOperationFunctionManager taskInstanceOperationFunctionManager; + + @Override + public TaskInstanceDispatchResponse dispatchTask(TaskInstanceDispatchRequest taskInstanceDispatchRequest) { + return taskInstanceOperationFunctionManager.getTaskInstanceDispatchOperationFunction() + .operate(taskInstanceDispatchRequest); + } + + @Override + public TaskInstanceKillResponse killTask(TaskInstanceKillRequest taskInstanceKillRequest) { + return taskInstanceOperationFunctionManager.getTaskInstanceKillOperationFunction() + .operate(taskInstanceKillRequest); + } + + @Override + public TaskInstancePauseResponse pauseTask(TaskInstancePauseRequest taskPauseRequest) { + return taskInstanceOperationFunctionManager.getTaskInstancePauseOperationFunction().operate(taskPauseRequest); + } + + @Override + public UpdateWorkflowHostResponse updateWorkflowInstanceHost(UpdateWorkflowHostRequest updateWorkflowHostRequest) { + return taskInstanceOperationFunctionManager.getUpdateWorkflowHostOperationFunction() + .operate(updateWorkflowHostRequest); + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerLogServiceImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerLogServiceImpl.java new file mode 100644 index 0000000000..405d79a0fb --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerLogServiceImpl.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.rpc; + +import static org.apache.dolphinscheduler.common.constants.Constants.APPID_COLLECT; +import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_COLLECT_WAY; + +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.extract.worker.IWorkerLogService; +import org.apache.dolphinscheduler.extract.worker.transportor.GetAppIdRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.GetAppIdResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; + +import java.io.File; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class WorkerLogServiceImpl implements IWorkerLogService { + + @Override + public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { + byte[] bytes = org.apache.dolphinscheduler.common.utils.LogUtils + .getFileContentBytes(taskInstanceLogFileDownloadRequest.getTaskInstanceLogAbsolutePath()); + // todo: if file not exists, return error result + return new TaskInstanceLogFileDownloadResponse(bytes); + } + + @Override + public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + List lines = org.apache.dolphinscheduler.common.utils.LogUtils.readPartFileContent( + taskInstanceLogPageQueryRequest.getTaskInstanceLogAbsolutePath(), + taskInstanceLogPageQueryRequest.getSkipLineNum(), + taskInstanceLogPageQueryRequest.getLimit()); + + String logContent = org.apache.dolphinscheduler.common.utils.LogUtils.rollViewLogLines(lines); + return new TaskInstanceLogPageQueryResponse(logContent); + } + + @Override + public GetAppIdResponse getAppId(GetAppIdRequest getAppIdRequest) { + TaskExecutionContext taskExecutionContext = + TaskExecutionContextCacheManager.getByTaskInstanceId(getAppIdRequest.getTaskInstanceId()); + String appInfoPath = taskExecutionContext.getAppInfoPath(); + String logPath = getAppIdRequest.getLogPath(); + List appIds = LogUtils.getAppIds(logPath, appInfoPath, + PropertyUtils.getString(APPID_COLLECT, DEFAULT_COLLECT_WAY)); + return new GetAppIdResponse(appIds); + } + + @Override + public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) { + File taskLogFile = new File(taskInstanceLogAbsolutePath); + try { + if (taskLogFile.exists()) { + taskLogFile.delete(); + } + } catch (Exception e) { + log.error("Remove TaskInstanceLog error", e); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java index a8b892fbaf..c72bec36e7 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java @@ -17,12 +17,10 @@ package org.apache.dolphinscheduler.server.worker.rpc; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.remote.command.BaseMessage; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; -import org.apache.dolphinscheduler.server.worker.message.MessageSender; +import org.apache.dolphinscheduler.server.worker.message.TaskInstanceExecutionEventSender; import java.util.HashMap; import java.util.List; @@ -44,9 +42,10 @@ public class WorkerMessageSender { private MessageRetryRunner messageRetryRunner; @Autowired - private List messageSenders; + private List messageSenders; - private Map messageSenderMap = new HashMap<>(); + private final Map messageSenderMap = + new HashMap<>(); @PostConstruct public void init() { @@ -56,30 +55,33 @@ public class WorkerMessageSender { // todo: use message rather than context public void sendMessageWithRetry(@NonNull TaskExecutionContext taskExecutionContext, - @NonNull MessageType messageType) { - MessageSender messageSender = messageSenderMap.get(messageType); + @NonNull ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType eventType) { + TaskInstanceExecutionEventSender messageSender = messageSenderMap.get(eventType); if (messageSender == null) { - throw new IllegalArgumentException("The messageType is invalidated, messageType: " + messageType); + throw new IllegalArgumentException("The messageType is invalidated, messageType: " + eventType); } - BaseMessage baseMessage = messageSender.buildMessage(taskExecutionContext); + ITaskInstanceExecutionEvent iTaskInstanceExecutionEvent = messageSender.buildEvent(taskExecutionContext); try { - messageRetryRunner.addRetryMessage(taskExecutionContext.getTaskInstanceId(), messageType, baseMessage); - messageSender.sendMessage(baseMessage); - } catch (RemotingException e) { - log.error("Send message error, messageType: {}, message: {}", messageType, baseMessage); + messageRetryRunner.addRetryMessage(taskExecutionContext.getTaskInstanceId(), iTaskInstanceExecutionEvent); + messageSender.sendEvent(iTaskInstanceExecutionEvent); + } catch (Exception e) { + log.error("Send message error, eventType: {}, event: {}", eventType, iTaskInstanceExecutionEvent); } } - public void sendMessage(@NonNull TaskExecutionContext taskExecutionContext, @NonNull MessageType messageType) { - MessageSender messageSender = messageSenderMap.get(messageType); + public void sendMessage(@NonNull TaskExecutionContext taskExecutionContext, + @NonNull ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType taskInstanceExecutionEventType) { + TaskInstanceExecutionEventSender messageSender = messageSenderMap.get(taskInstanceExecutionEventType); if (messageSender == null) { - throw new IllegalArgumentException("The messageType is invalidated, messageType: " + messageType); + throw new IllegalArgumentException( + "The eventType is invalidated, eventType: " + taskInstanceExecutionEventType); } - BaseMessage baseMessage = messageSender.buildMessage(taskExecutionContext); + ITaskInstanceExecutionEvent iTaskInstanceExecutionEvent = messageSender.buildEvent(taskExecutionContext); try { - messageSender.sendMessage(baseMessage); - } catch (RemotingException e) { - log.error("Send message error, messageType: {}, message: {}", messageType, baseMessage); + messageSender.sendEvent(iTaskInstanceExecutionEvent); + } catch (Exception e) { + log.error("Send message error, eventType: {}, event: {}", taskInstanceExecutionEventType, + iTaskInstanceExecutionEvent); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java deleted file mode 100644 index e7a025546e..0000000000 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.rpc; - -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; - -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; - -/** - * This rpc client is only used to send message, will not receive message, all response message should send to {@link WorkerRpcServer}. - */ -@Component -@Slf4j -public class WorkerRpcClient implements AutoCloseable { - - @Autowired - @Lazy - private List workerRpcProcessors; - - @Autowired - private WorkerConfig workerConfig; - - private NettyRemotingClient nettyRemotingClient; - - public void start() { - log.info("Worker rpc client starting"); - this.nettyRemotingClient = new NettyRemotingClient(workerConfig.getWorkerRpcClientConfig()); - // we only use the client to handle the ack message, we can optimize this, send ack to the nettyServer. - for (WorkerRpcProcessor workerRpcProcessor : workerRpcProcessors) { - this.nettyRemotingClient.registerProcessor(workerRpcProcessor); - } - log.info("Worker rpc client started"); - } - - public void send(Host host, Message message) throws RemotingException { - nettyRemotingClient.send(host, message); - } - - public void close() { - log.info("Worker rpc client closing"); - nettyRemotingClient.close(); - log.info("Worker rpc client closed"); - } -} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java index cf81ad4f55..a66b064d98 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java @@ -17,49 +17,36 @@ package org.apache.dolphinscheduler.server.worker.rpc; -import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; +import org.apache.dolphinscheduler.extract.base.NettyRemotingServerFactory; +import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import java.io.Closeable; -import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -@Service @Slf4j -public class WorkerRpcServer implements Closeable { - - @Autowired - private List workerRpcProcessors; - - @Autowired - private WorkerConfig workerConfig; +@Service +public class WorkerRpcServer extends SpringServerMethodInvokerDiscovery implements Closeable { - private NettyRemotingServer nettyRemotingServer; + public WorkerRpcServer(WorkerConfig workerConfig) { + super(NettyRemotingServerFactory.buildNettyRemotingServer(new NettyServerConfig(workerConfig.getListenPort()))); + } public void start() { - log.info("Worker rpc server starting..."); - NettyServerConfig serverConfig = workerConfig.getWorkerRpcServerConfig(); - serverConfig.setListenPort(workerConfig.getListenPort()); - nettyRemotingServer = new NettyRemotingServer(serverConfig); - for (WorkerRpcProcessor workerRpcProcessor : workerRpcProcessors) { - nettyRemotingServer.registerProcessor(workerRpcProcessor); - log.info("Success register WorkerRpcProcessor: {}", workerRpcProcessor.getClass().getName()); - } - this.nettyRemotingServer.start(); - log.info("Worker rpc server started..."); + log.info("WorkerRpcServer starting..."); + nettyRemotingServer.start(); + log.info("WorkerRpcServer started..."); } @Override public void close() { - log.info("Worker rpc server closing"); - this.nettyRemotingServer.close(); - log.info("Worker rpc server closed"); + log.info("WorkerRpcServer closing"); + nettyRemotingServer.close(); + log.info("WorkerRpcServer closed"); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java index b8b2328774..82b91e16ba 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import javax.annotation.Nullable; @@ -36,14 +35,12 @@ public class DefaultWorkerDelayTaskExecuteRunnable extends WorkerDelayTaskExecut public DefaultWorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull WorkerRpcClient workerRpcClient, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { super(taskExecutionContext, workerConfig, workerMessageSender, - workerRpcClient, taskPluginManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java index 5fd6ae7b53..631b29cc8e 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import javax.annotation.Nullable; @@ -36,14 +35,12 @@ public class DefaultWorkerDelayTaskExecuteRunnableFactory protected DefaultWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull WorkerRpcClient workerRpcClient, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { super(taskExecutionContext, workerConfig, workerMessageSender, - workerRpcClient, taskPluginManager, storageOperate, workerRegistryClient); @@ -55,7 +52,6 @@ public class DefaultWorkerDelayTaskExecuteRunnableFactory taskExecutionContext, workerConfig, workerMessageSender, - workerRpcClient, taskPluginManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueue.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueue.java new file mode 100644 index 0000000000..2b894913a7 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueue.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner; + +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.config.TaskExecuteThreadsFullPolicy; +import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; +import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class GlobalTaskInstanceDispatchQueue { + + private final WorkerConfig workerConfig; + + private final BlockingQueue blockingQueue; + + public GlobalTaskInstanceDispatchQueue(WorkerConfig workerConfig) { + this.workerConfig = workerConfig; + this.blockingQueue = new ArrayBlockingQueue<>(workerConfig.getExecThreads()); + } + + public boolean addDispatchTask(TaskExecutionContext taskExecutionContext) { + if (workerConfig.getTaskExecuteThreadsFullPolicy() == TaskExecuteThreadsFullPolicy.CONTINUE) { + return blockingQueue.offer(taskExecutionContext); + } + + if (blockingQueue.size() > getQueueSize()) { + log.warn("Wait submit queue is full, will retry submit task later"); + WorkerServerMetrics.incWorkerSubmitQueueIsFullCount(); + return false; + } + return blockingQueue.offer(taskExecutionContext); + } + + public TaskExecutionContext take() throws InterruptedException { + return blockingQueue.take(); + } + + public void clearTask() { + blockingQueue.clear(); + } + + public int getQueueSize() { + return workerConfig.getExecThreads(); + } + +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueueLooper.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueueLooper.java new file mode 100644 index 0000000000..eb0e33d389 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/GlobalTaskInstanceDispatchQueueLooper.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner; + +import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; +import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; +import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class GlobalTaskInstanceDispatchQueueLooper extends BaseDaemonThread { + + @Autowired + private GlobalTaskInstanceDispatchQueue globalTaskInstanceDispatchQueue; + + @Autowired + private WorkerConfig workerConfig; + + @Autowired + private WorkerMessageSender workerMessageSender; + + @Autowired + private TaskPluginManager taskPluginManager; + + @Autowired + private WorkerManagerThread workerManager; + + @Autowired(required = false) + private StorageOperate storageOperate; + + @Autowired + private WorkerRegistryClient workerRegistryClient; + + protected GlobalTaskInstanceDispatchQueueLooper() { + super("GlobalTaskDispatchQueueLooper"); + } + + public synchronized void start() { + log.info("GlobalTaskDispatchQueueLooper starting"); + super.start(); + log.info("GlobalTaskDispatchQueueLooper started"); + } + + public void run() { + while (true) { + try { + TaskExecutionContext taskExecutionContext = globalTaskInstanceDispatchQueue.take(); + LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath()); + LogUtils.setTaskInstanceIdMDC(taskExecutionContext.getTaskInstanceId()); + // delay task process + long remainTime = + DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), + taskExecutionContext.getDelayTime() * 60L); + if (remainTime > 0) { + log.info("Current taskInstance is choose delay execution, delay time: {}s", remainTime); + taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.DELAY_EXECUTION); + // todo: use delay running event + workerMessageSender.sendMessage(taskExecutionContext, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.FINISH); + } + WorkerDelayTaskExecuteRunnable workerTaskExecuteRunnable = WorkerTaskExecuteRunnableFactoryBuilder + .createWorkerDelayTaskExecuteRunnableFactory( + taskExecutionContext, + workerConfig, + workerMessageSender, + taskPluginManager, + storageOperate, + workerRegistryClient) + .createWorkerTaskExecuteRunnable(); + if (workerManager.offer(workerTaskExecuteRunnable)) { + log.info("Success submit WorkerDelayTaskExecuteRunnable to WorkerManagerThread's waiting queue"); + } + } catch (InterruptedException e) { + log.error("GlobalTaskDispatchQueueLooper interrupted"); + break; + } catch (Exception ex) { + log.error("GlobalTaskDispatchQueueLooper error", ex); + } finally { + LogUtils.removeTaskInstanceIdMDC(); + LogUtils.removeTaskInstanceLogFullPathMDC(); + } + } + } + +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskCallbackImpl.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskCallbackImpl.java index b6a8f0fe72..be8a62a6c8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskCallbackImpl.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskCallbackImpl.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.server.worker.runner; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; -import org.apache.dolphinscheduler.remote.command.MessageType; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import lombok.Builder; @@ -52,7 +52,8 @@ public class TaskCallbackImpl implements TaskCallBack { log.info("send remote application info {}", applicationInfo); taskExecutionContext.setAppIds(applicationInfo.getAppIds()); - workerMessageSender.sendMessageWithRetry(taskExecutionContext, MessageType.TASK_EXECUTE_RUNNING_MESSAGE); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.RUNNING_INFO); } @Override @@ -64,7 +65,8 @@ public class TaskCallbackImpl implements TaskCallBack { return; } - workerMessageSender.sendMessageWithRetry(taskExecutionContext, MessageType.TASK_UPDATE_RUNTIME_MESSAGE); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.RUNNING_INFO); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java index 2bb09ac07d..07d85dc57b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; @@ -38,14 +37,12 @@ public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRu protected WorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull WorkerRpcClient workerRpcClient, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { super(taskExecutionContext, workerConfig, workerMessageSender, - workerRpcClient, taskPluginManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java index ce8635ed48..f2d39835b1 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import javax.annotation.Nullable; @@ -36,7 +35,6 @@ public abstract class WorkerDelayTaskExecuteRunnableFactory workerExecThreads) { - log.warn("Wait submit queue is full, will retry submit task later"); - WorkerServerMetrics.incWorkerSubmitQueueIsFullCount(); - // if waitSubmitQueue is full, it will wait 1s, then try add - ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); - if (waitSubmitQueue.size() > workerExecThreads) { - return false; - } - } - return waitSubmitQueue.offer(workerDelayTaskExecuteRunnable); + return waitSubmitQueue.add(workerDelayTaskExecuteRunnable); } public void start() { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java index 5e985c628c..5fadf27ef0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java @@ -26,6 +26,12 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.log.SensitiveDataConverter; import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.extract.alert.IAlertOperator; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest; +import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.base.utils.Host; +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; @@ -41,14 +47,9 @@ import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader; import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequest; -import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import org.apache.dolphinscheduler.server.worker.utils.TaskExecutionCheckerUtils; import org.apache.dolphinscheduler.server.worker.utils.TaskFilesTransferUtils; @@ -75,7 +76,6 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { protected final WorkerMessageSender workerMessageSender; protected final TaskPluginManager taskPluginManager; protected final @Nullable StorageOperate storageOperate; - protected final WorkerRpcClient workerRpcClient; protected final WorkerRegistryClient workerRegistryClient; protected @Nullable AbstractTask task; @@ -84,14 +84,12 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { @NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull WorkerRpcClient workerRpcClient, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.workerRpcClient = workerRpcClient; this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; this.workerRegistryClient = workerRegistryClient; @@ -121,7 +119,8 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.FAILURE); taskExecutionContext.setEndTime(System.currentTimeMillis()); - workerMessageSender.sendMessageWithRetry(taskExecutionContext, MessageType.TASK_EXECUTE_RESULT_MESSAGE); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.FINISH); log.info("Get a exception when execute the task, will send the task status: {} to master: {}", TaskExecutionStatus.FAILURE.name(), taskExecutionContext.getHost()); @@ -158,7 +157,7 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { taskExecutionContext.setEndTime(System.currentTimeMillis()); TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); workerMessageSender.sendMessageWithRetry(taskExecutionContext, - MessageType.TASK_EXECUTE_RESULT_MESSAGE); + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.FINISH); log.info( "The current execute mode is dry run, will stop the subsequent process and set the taskInstance status to success"); return; @@ -204,7 +203,8 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { protected void beforeExecute() { taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.RUNNING_EXECUTION); - workerMessageSender.sendMessageWithRetry(taskExecutionContext, MessageType.TASK_EXECUTE_RUNNING_MESSAGE); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.RUNNING); log.info("Send task status {} master: {}", TaskExecutionStatus.RUNNING_EXECUTION.name(), taskExecutionContext.getHost()); @@ -249,20 +249,20 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { } Host alertServerAddress = alertServerAddressOptional.get(); - TaskExecutionStatus status = task.getExitStatus(); TaskAlertInfo taskAlertInfo = task.getTaskAlertInfo(); - int strategy = - status == TaskExecutionStatus.SUCCESS ? WarningType.SUCCESS.getCode() : WarningType.FAILURE.getCode(); - AlertSendRequest alertCommand = new AlertSendRequest( + AlertSendRequest alertSendRequest = new AlertSendRequest( taskAlertInfo.getAlertGroupId(), taskAlertInfo.getTitle(), taskAlertInfo.getContent(), - strategy); + task.getExitStatus() == TaskExecutionStatus.SUCCESS ? WarningType.SUCCESS.getCode() + : WarningType.FAILURE.getCode()); try { - workerRpcClient.send(alertServerAddress, alertCommand.convert2Command()); - log.info("Send alert to: {} successfully", alertServerAddress); - } catch (RemotingException e) { - log.error("Send alert to: {} failed, alertCommand: {}", alertServerAddress, alertCommand, e); + IAlertOperator alertOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance() + .getProxyClient(alertServerAddress.getAddress(), IAlertOperator.class); + AlertSendResponse alertSendResponse = alertOperator.sendAlert(alertSendRequest); + log.info("Send alert to: {} successfully, response: {}", alertServerAddress, alertSendResponse); + } catch (Exception e) { + log.error("Send alert: {} to: {} failed", alertSendRequest, alertServerAddress, e); } } @@ -278,7 +278,8 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { log.info("Upload output files: {} successfully", TaskFilesTransferUtils.getFileLocalParams(taskExecutionContext, Direct.OUT)); - workerMessageSender.sendMessageWithRetry(taskExecutionContext, MessageType.TASK_EXECUTE_RESULT_MESSAGE); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.FINISH); log.info("Send task execute status: {} to master : {}", taskExecutionContext.getCurrentExecutionStatus().name(), taskExecutionContext.getHost()); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java index 33b1a9b755..603462e5b0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import javax.annotation.Nullable; @@ -36,14 +35,12 @@ public class WorkerTaskExecuteRunnableFactoryBuilder { public static WorkerDelayTaskExecuteRunnableFactory createWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @NonNull WorkerRpcClient workerRpcClient, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate, @NonNull WorkerRegistryClient workerRegistryClient) { return new DefaultWorkerDelayTaskExecuteRunnableFactory(taskExecutionContext, workerConfig, workerMessageSender, - workerRpcClient, taskPluginManager, storageOperate, workerRegistryClient); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/ITaskInstanceExecutionEventAckListenFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/ITaskInstanceExecutionEventAckListenFunction.java new file mode 100644 index 0000000000..303a19d663 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/ITaskInstanceExecutionEventAckListenFunction.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.listener; + +public interface ITaskInstanceExecutionEventAckListenFunction { + + void handleTaskInstanceExecutionEventAck(E e); + +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionEventAckListenFunctionManager.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionEventAckListenFunctionManager.java new file mode 100644 index 0000000000..b4423f4880 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionEventAckListenFunctionManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.listener; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionEventAckListenFunctionManager { + + @Autowired + private TaskInstanceExecutionRunningEventAckListenFunction taskInstanceExecutionRunningEventAckListenFunction; + + @Autowired + private TaskInstanceExecutionFinishEventAckListenFunction taskInstanceExecutionFinishEventAckListenFunction; + + @Autowired + private TaskInstanceExecutionInfoEventAckListenFunction taskInstanceExecutionInfoEventAckListenFunction; + + public TaskInstanceExecutionRunningEventAckListenFunction getTaskInstanceExecutionRunningEventAckListenFunction() { + return taskInstanceExecutionRunningEventAckListenFunction; + } + + public TaskInstanceExecutionFinishEventAckListenFunction getTaskInstanceExecutionFinishEventAckListenFunction() { + return taskInstanceExecutionFinishEventAckListenFunction; + } + + public TaskInstanceExecutionInfoEventAckListenFunction getTaskInstanceExecutionInfoEventAckListenFunction() { + return taskInstanceExecutionInfoEventAckListenFunction; + } + +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionFinishEventAckListenFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionFinishEventAckListenFunction.java new file mode 100644 index 0000000000..ad7892bc7a --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionFinishEventAckListenFunction.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.listener; + +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionFinishEventAck; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionFinishEventAckListenFunction + implements + ITaskInstanceExecutionEventAckListenFunction { + + @Autowired + private MessageRetryRunner messageRetryRunner; + + @Override + public void handleTaskInstanceExecutionEventAck(TaskInstanceExecutionFinishEventAck taskInstanceExecutionFinishEventAck) { + try { + final int taskInstanceId = taskInstanceExecutionFinishEventAck.getTaskInstanceId(); + LogUtils.setTaskInstanceIdMDC(taskInstanceId); + log.info("Receive TaskInstanceExecutionFinishEventAck: {}", taskInstanceExecutionFinishEventAck); + if (taskInstanceExecutionFinishEventAck.isSuccess()) { + messageRetryRunner.removeRetryMessage(taskInstanceId, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.FINISH); + } else { + // todo: if the ack is failed, it needn't sent to worker + log.warn("TaskInstanceExecutionFinishEventAck failed: {}", taskInstanceExecutionFinishEventAck); + } + } finally { + LogUtils.removeTaskInstanceIdMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionInfoEventAckListenFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionInfoEventAckListenFunction.java new file mode 100644 index 0000000000..971343103a --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionInfoEventAckListenFunction.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.listener; + +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionInfoEventAck; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; + +import javax.annotation.Resource; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionInfoEventAckListenFunction + implements + ITaskInstanceExecutionEventAckListenFunction { + + @Resource + private MessageRetryRunner messageRetryRunner; + + @Override + public void handleTaskInstanceExecutionEventAck(TaskInstanceExecutionInfoEventAck taskInstanceExecutionInfoEventAck) { + try { + final int taskInstanceId = taskInstanceExecutionInfoEventAck.getTaskInstanceId(); + LogUtils.setTaskInstanceIdMDC(taskInstanceId); + log.info("Receive TaskInstanceExecutionInfoEventAck: {}", taskInstanceExecutionInfoEventAck); + if (taskInstanceExecutionInfoEventAck.isSuccess()) { + messageRetryRunner.removeRetryMessage(taskInstanceId, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.RUNNING_INFO); + } else { + // todo: if the ack is failed, it needn't sent to worker + log.warn("TaskInstanceExecutionInfoEventAck failed: {}", taskInstanceExecutionInfoEventAck); + } + } finally { + LogUtils.removeTaskInstanceIdMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionRunningEventAckListenFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionRunningEventAckListenFunction.java new file mode 100644 index 0000000000..9d6de78e02 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/listener/TaskInstanceExecutionRunningEventAckListenFunction.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.listener; + +import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceExecutionRunningEventAckListenFunction + implements + ITaskInstanceExecutionEventAckListenFunction { + + @Autowired + private MessageRetryRunner messageRetryRunner; + + @Override + public void handleTaskInstanceExecutionEventAck(TaskInstanceExecutionRunningEventAck taskInstanceExecutionRunningEventAck) { + try { + final int taskInstanceId = taskInstanceExecutionRunningEventAck.getTaskInstanceId(); + LogUtils.setTaskInstanceIdMDC(taskInstanceId); + log.info("Receive TaskInstanceExecutionRunningEventAck: {}", taskInstanceExecutionRunningEventAck); + if (taskInstanceExecutionRunningEventAck.isSuccess()) { + messageRetryRunner.removeRetryMessage(taskInstanceId, + ITaskInstanceExecutionEvent.TaskInstanceExecutionEventType.RUNNING); + } else { + log.warn("TaskInstanceExecutionRunningEventAck failed: {}", taskInstanceExecutionRunningEventAck); + } + } finally { + LogUtils.removeTaskInstanceIdMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java new file mode 100644 index 0000000000..e8865c985c --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.operator; + +public interface ITaskInstanceOperationFunction { + + Y operate(X x); + +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceDispatchOperationFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceDispatchOperationFunction.java new file mode 100644 index 0000000000..3ff110c459 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceDispatchOperationFunction.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.operator; + +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchResponse; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; +import org.apache.dolphinscheduler.server.worker.metrics.TaskMetrics; +import org.apache.dolphinscheduler.server.worker.runner.GlobalTaskInstanceDispatchQueue; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstanceDispatchOperationFunction + implements + ITaskInstanceOperationFunction { + + @Autowired + private WorkerConfig workerConfig; + + @Autowired + private GlobalTaskInstanceDispatchQueue globalTaskInstanceDispatchQueue; + + @Override + public TaskInstanceDispatchResponse operate(TaskInstanceDispatchRequest taskInstanceDispatchRequest) { + log.info("Receive TaskInstanceDispatchRequest: {}", taskInstanceDispatchRequest); + TaskExecutionContext taskExecutionContext = taskInstanceDispatchRequest.getTaskExecutionContext(); + try { + // set cache, it will be used when kill task + TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); + taskExecutionContext.setHost(workerConfig.getWorkerAddress()); + taskExecutionContext.setLogPath(LogUtils.getTaskInstanceLogFullPath(taskExecutionContext)); + + LogUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); + TaskMetrics.incrTaskTypeExecuteCount(taskExecutionContext.getTaskType()); + + if (!globalTaskInstanceDispatchQueue.addDispatchTask(taskExecutionContext)) { + log.error("Submit task: {} to wait queue error, current queue size: {} is full", + taskExecutionContext.getTaskName(), workerConfig.getExecThreads()); + return TaskInstanceDispatchResponse.failed(taskExecutionContext.getTaskInstanceId(), + "WorkerManagerThread is full"); + } else { + log.info("Submit task: {} to wait queue success", taskExecutionContext.getTaskName()); + return TaskInstanceDispatchResponse.success(taskExecutionContext.getTaskInstanceId()); + } + } finally { + LogUtils.removeWorkflowAndTaskInstanceIdMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskKillProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java similarity index 59% rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskKillProcessor.java rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java index 64a9240997..347a52fa50 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskKillProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceKillOperationFunction.java @@ -15,29 +15,22 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.processor; +package org.apache.dolphinscheduler.server.worker.runner.operator; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.MessageType; -import org.apache.dolphinscheduler.remote.command.task.TaskKillRequest; -import org.apache.dolphinscheduler.remote.command.task.TaskKillResponse; -import org.apache.dolphinscheduler.remote.processor.WorkerRpcProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; -import java.util.Arrays; - import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -45,15 +38,11 @@ import org.springframework.stereotype.Component; import com.google.common.base.Strings; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFutureListener; - -/** - * task kill processor - */ @Slf4j @Component -public class WorkerTaskKillProcessor implements WorkerRpcProcessor { +public class TaskInstanceKillOperationFunction + implements + ITaskInstanceOperationFunction { @Autowired private WorkerManagerThread workerManager; @@ -61,29 +50,20 @@ public class WorkerTaskKillProcessor implements WorkerRpcProcessor { @Autowired private MessageRetryRunner messageRetryRunner; - /** - * task kill process - * - * @param channel channel channel - * @param message command command - */ @Override - public void process(Channel channel, Message message) { - TaskKillRequest killCommand = JSONUtils.parseObject(message.getBody(), TaskKillRequest.class); - if (killCommand == null) { - log.error("task kill request command is null"); - return; - } - log.info("task kill command : {}", killCommand); + public TaskInstanceKillResponse operate(TaskInstanceKillRequest taskInstanceKillRequest) { + log.info("Receive TaskInstanceKillRequest: {}", taskInstanceKillRequest); - int taskInstanceId = killCommand.getTaskInstanceId(); - try (LogUtils.MDCAutoClosableContext mdcAutoClosableContext = LogUtils.setTaskInstanceIdMDC(taskInstanceId)) { + int taskInstanceId = taskInstanceKillRequest.getTaskInstanceId(); + try { + LogUtils.setTaskInstanceIdMDC(taskInstanceId); TaskExecutionContext taskExecutionContext = TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); if (taskExecutionContext == null) { - log.error("taskRequest cache is null, taskInstanceId: {}", killCommand.getTaskInstanceId()); - return; + log.error("Cannot find TaskExecutionContext for taskInstance: {}", taskInstanceId); + return TaskInstanceKillResponse.fail("Cannot find TaskExecutionContext"); } + LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath()); boolean result = doKill(taskExecutionContext); this.cancelApplication(taskInstanceId); @@ -93,49 +73,22 @@ public class WorkerTaskKillProcessor implements WorkerRpcProcessor { workerManager.killTaskBeforeExecuteByInstanceId(taskInstanceId); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.KILL); TaskExecutionContextCacheManager.removeByTaskInstanceId(taskInstanceId); - sendTaskKillResponseCommand(channel, message.getOpaque(), taskExecutionContext); - log.info("the task has not been executed and has been cancelled, task id:{}", taskInstanceId); - return; + log.info("The task has not been executed and has been cancelled, task id:{}", taskInstanceId); + return TaskInstanceKillResponse.success(taskExecutionContext); } - taskExecutionContext.setCurrentExecutionStatus( - result ? TaskExecutionStatus.SUCCESS : TaskExecutionStatus.FAILURE); - sendTaskKillResponseCommand(channel, message.getOpaque(), taskExecutionContext); + taskExecutionContext + .setCurrentExecutionStatus(result ? TaskExecutionStatus.SUCCESS : TaskExecutionStatus.FAILURE); TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); messageRetryRunner.removeRetryMessages(taskExecutionContext.getTaskInstanceId()); - - log.info("remove REMOTE_CHANNELS, task instance id:{}", killCommand.getTaskInstanceId()); + return TaskInstanceKillResponse.success(taskExecutionContext); + } finally { + LogUtils.removeTaskInstanceIdMDC(); + LogUtils.removeTaskInstanceLogFullPathMDC(); } } - @Override - public MessageType getCommandType() { - return MessageType.TASK_KILL_REQUEST; - } - - private void sendTaskKillResponseCommand(Channel channel, long opaque, TaskExecutionContext taskExecutionContext) { - TaskKillResponse taskKillResponse = new TaskKillResponse(); - taskKillResponse.setStatus(taskExecutionContext.getCurrentExecutionStatus()); - if (taskExecutionContext.getAppIds() != null) { - taskKillResponse - .setAppIds(Arrays.asList(taskExecutionContext.getAppIds().split(TaskConstants.COMMA))); - } - taskKillResponse.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - taskKillResponse.setHost(taskExecutionContext.getHost()); - taskKillResponse.setProcessId(taskExecutionContext.getProcessId()); - channel.writeAndFlush(taskKillResponse.convert2Command(opaque)).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - log.error("Submit kill response to master error, kill command: {}", taskKillResponse); - } - }); - } - - /** - * do kill - * - * @return kill result - */ private boolean doKill(TaskExecutionContext taskExecutionContext) { // kill system process boolean processFlag = killProcess(taskExecutionContext.getTenantCode(), taskExecutionContext.getProcessId()); @@ -149,10 +102,6 @@ public class WorkerTaskKillProcessor implements WorkerRpcProcessor { return processFlag; } - /** - * kill task by cancel application - * @param taskInstanceId - */ protected void cancelApplication(int taskInstanceId) { WorkerTaskExecuteRunnable workerTaskExecuteRunnable = workerManager.getTaskExecuteThread(taskInstanceId); if (workerTaskExecuteRunnable == null) { @@ -172,11 +121,6 @@ public class WorkerTaskKillProcessor implements WorkerRpcProcessor { log.info("kill task by cancelApplication, task id:{}", taskInstanceId); } - /** - * kill system process - * @param tenantCode - * @param processId - */ protected boolean killProcess(String tenantCode, Integer processId) { // todo: directly interrupt the process boolean processFlag = true; @@ -197,5 +141,4 @@ public class WorkerTaskKillProcessor implements WorkerRpcProcessor { } return processFlag; } - } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionManager.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionManager.java new file mode 100644 index 0000000000..99ae193b47 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionManager.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.operator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TaskInstanceOperationFunctionManager { + + @Autowired + private TaskInstanceKillOperationFunction taskInstanceKillOperationFunction; + + @Autowired + private UpdateWorkflowHostOperationFunction updateWorkflowHostOperationFunction; + + @Autowired + private TaskInstanceDispatchOperationFunction taskInstanceDispatchOperationFunction; + + @Autowired + private TaskInstancePauseOperationFunction taskInstancePauseOperationFunction; + + public TaskInstanceKillOperationFunction getTaskInstanceKillOperationFunction() { + return taskInstanceKillOperationFunction; + } + + public UpdateWorkflowHostOperationFunction getUpdateWorkflowHostOperationFunction() { + return updateWorkflowHostOperationFunction; + } + + public TaskInstanceDispatchOperationFunction getTaskInstanceDispatchOperationFunction() { + return taskInstanceDispatchOperationFunction; + } + + public TaskInstancePauseOperationFunction getTaskInstancePauseOperationFunction() { + return taskInstancePauseOperationFunction; + } + +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstancePauseOperationFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstancePauseOperationFunction.java new file mode 100644 index 0000000000..5ff62a5477 --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstancePauseOperationFunction.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.operator; + +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class TaskInstancePauseOperationFunction + implements + ITaskInstanceOperationFunction { + + @Override + public TaskInstancePauseResponse operate(TaskInstancePauseRequest taskInstancePauseRequest) { + try { + LogUtils.setTaskInstanceIdMDC(taskInstancePauseRequest.getTaskInstanceId()); + log.info("Receive TaskInstancePauseRequest: {}", taskInstancePauseRequest); + log.warn("TaskInstancePauseOperationFunction is not support for worker task yet!"); + return TaskInstancePauseResponse.success(); + } finally { + LogUtils.removeTaskInstanceIdMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/UpdateWorkflowHostOperationFunction.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/UpdateWorkflowHostOperationFunction.java new file mode 100644 index 0000000000..6d31feff7e --- /dev/null +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/UpdateWorkflowHostOperationFunction.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner.operator; + +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest; +import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; +import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; +import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class UpdateWorkflowHostOperationFunction + implements + ITaskInstanceOperationFunction { + + @Autowired + private MessageRetryRunner messageRetryRunner; + + @Override + public UpdateWorkflowHostResponse operate(UpdateWorkflowHostRequest updateWorkflowHostRequest) { + try { + final int taskInstanceId = updateWorkflowHostRequest.getTaskInstanceId(); + final String workflowHost = updateWorkflowHostRequest.getWorkflowHost(); + + LogUtils.setTaskInstanceIdMDC(taskInstanceId); + log.info("Received UpdateWorkflowHostRequest: {}", updateWorkflowHostRequest); + + TaskExecutionContext taskExecutionContext = + TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); + if (taskExecutionContext == null) { + log.error("Cannot find the taskExecutionContext for taskInstance : {}", taskInstanceId); + return UpdateWorkflowHostResponse.failed("Cannot find the taskExecutionContext"); + } + + LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath()); + taskExecutionContext.setWorkflowInstanceHost(workflowHost); + messageRetryRunner.updateMessageHost(taskInstanceId, workflowHost); + log.info("Success update workflow host: {} for taskInstance: {}", workflowHost, taskInstanceId); + return UpdateWorkflowHostResponse.success(); + } finally { + LogUtils.removeTaskInstanceIdMDC(); + LogUtils.removeTaskInstanceLogFullPathMDC(); + } + } +} diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskDispatchProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskDispatchProcessorTest.java deleted file mode 100644 index c65d66cb66..0000000000 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/WorkerTaskDispatchProcessorTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.server.worker.processor; - -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; -import org.apache.dolphinscheduler.remote.command.Message; -import org.apache.dolphinscheduler.remote.command.task.TaskDispatchRequest; -import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; -import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; -import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import io.netty.channel.Channel; - -/** - * test task execute processor - */ -@ExtendWith(MockitoExtension.class) -public class WorkerTaskDispatchProcessorTest { - - @InjectMocks - private WorkerTaskDispatchProcessor workerTaskDispatchProcessor; - - @Mock - private WorkerConfig workerConfig; - - @Mock - private WorkerMessageSender workerMessageSender; - - @Mock - private WorkerRpcClient workerRpcClient; - - @Mock - private TaskPluginManager taskPluginManager; - - @Mock - private WorkerManagerThread workerManagerThread; - - @Mock - private StorageOperate storageOperate; - - @Mock - private WorkerRegistryClient workerRegistryClient; - - @Test - public void process() { - Channel channel = Mockito.mock(Channel.class); - TaskExecutionContext taskExecutionContext = getTaskExecutionContext(); - Message dispatchMessage = createDispatchCommand(taskExecutionContext); - workerTaskDispatchProcessor.process(channel, dispatchMessage); - - Mockito.verify(workerManagerThread, Mockito.atMostOnce()).offer(Mockito.any()); - } - - public Message createDispatchCommand(TaskExecutionContext taskExecutionContext) { - return new TaskDispatchRequest(taskExecutionContext).convert2Command(); - } - - public TaskExecutionContext getTaskExecutionContext() { - TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); - taskExecutionContext.setProcessId(12345); - taskExecutionContext.setProcessInstanceId(1); - taskExecutionContext.setTaskInstanceId(1); - taskExecutionContext.setProcessDefineCode(1L); - taskExecutionContext.setProcessDefineVersion(1); - taskExecutionContext.setTaskType("SQL"); - taskExecutionContext.setFirstSubmitTime(System.currentTimeMillis()); - taskExecutionContext.setDelayTime(0); - taskExecutionContext.setLogPath("/tmp/test.log"); - taskExecutionContext.setHost("localhost"); - taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); - return taskExecutionContext; - } -} diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java index 29d30167d3..79d04edae1 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -41,8 +40,6 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); - private WorkerRpcClient alertClientService = Mockito.mock(WorkerRpcClient.class); - private TaskPluginManager taskPluginManager = Mockito.mock(TaskPluginManager.class); private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); @@ -61,7 +58,6 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { taskExecutionContext, workerConfig, workerMessageSender, - alertClientService, taskPluginManager, storageOperate, workerRegistryClient); @@ -86,7 +82,6 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { taskExecutionContext, workerConfig, workerMessageSender, - alertClientService, taskPluginManager, storageOperate, workerRegistryClient); diff --git a/pom.xml b/pom.xml index ab539c6e31..a6c5f185e5 100755 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,6 @@ dolphinscheduler-aop dolphinscheduler-dao dolphinscheduler-dist - dolphinscheduler-remote dolphinscheduler-service dolphinscheduler-microbench dolphinscheduler-data-quality @@ -56,6 +55,7 @@ dolphinscheduler-ui dolphinscheduler-scheduler-plugin dolphinscheduler-storage-plugin + dolphinscheduler-extract @@ -144,11 +144,7 @@ dolphinscheduler-dao ${project.version} - - org.apache.dolphinscheduler - dolphinscheduler-remote - ${project.version} - + org.apache.dolphinscheduler dolphinscheduler-service @@ -242,6 +238,26 @@ dolphinscheduler-storage-all ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-extract-base + ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-extract-master + ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-extract-worker + ${project.version} + + + org.apache.dolphinscheduler + dolphinscheduler-extract-alert + ${project.version} + org.junit diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 23fb7919d9..303c3d4e6f 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -29,7 +29,6 @@ bcprov-jdk15on-1.69.jar bcutil-jdk15on-1.69.jar bonecp-0.8.0.RELEASE.jar bucket4j-core-6.2.0.jar -byte-buddy-1.9.16.jar caffeine-2.9.3.jar checker-qual-3.12.0.jar checker-qual-3.19.0.jar @@ -133,7 +132,6 @@ jakarta.websocket-api-1.1.2.jar jakarta.xml.bind-api-2.3.3.jar jamon-runtime-2.3.1.jar janino-3.0.16.jar -javassist-3.28.0-GA.jar javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar javax.el-3.0.0.jar @@ -269,15 +267,10 @@ postgresql-42.4.1.jar presto-jdbc-0.238.1.jar profiles-2.17.282.jar protocol-core-2.17.282.jar -protostuff-api-1.7.2.jar -protostuff-collectionschema-1.7.2.jar -protostuff-core-1.7.2.jar -protostuff-runtime-1.7.2.jar py4j-0.10.9.jar quartz-2.3.2.jar re2j-1.1.jar reactive-streams-1.0.4.jar -reflections-0.10.2.jar regions-2.17.282.jar reload4j-1.2.18.3.jar sdk-core-2.17.282.jar