From 022e4886be9a170052f80d56f08c521a5b1ceb5a Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Sat, 4 Jun 2022 00:18:01 +0800 Subject: [PATCH] Remove quartz at WorkerServer (#10358) * Remove quartz at WorkerServer * move k8s and permission from dolphinscheduler-service to dolphinscheduler-api --- .../api}/k8s/K8sClientService.java | 0 .../dolphinscheduler/api}/k8s/K8sManager.java | 0 .../api}/permission/PermissionCheck.java | 0 .../ResourcePermissionCheckService.java | 0 .../ResourcePermissionCheckServiceImpl.java | 0 .../api}/k8s/K8sManagerTest.java | 0 .../ResourcePermissionCheckServiceTest.java | 0 dolphinscheduler-service/pom.xml | 5 - dolphinscheduler-worker/pom.xml | 6 + .../server/worker/WorkerServer.java | 14 +- .../TaskCallbackServiceTestConfig.java | 156 ------------------ 11 files changed, 19 insertions(+), 162 deletions(-) rename {dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api}/k8s/K8sClientService.java (100%) rename {dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api}/k8s/K8sManager.java (100%) rename {dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api}/permission/PermissionCheck.java (100%) rename {dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api}/permission/ResourcePermissionCheckService.java (100%) rename {dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api}/permission/ResourcePermissionCheckServiceImpl.java (100%) rename {dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api}/k8s/K8sManagerTest.java (100%) rename {dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service => dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api}/permission/ResourcePermissionCheckServiceTest.java (100%) diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/k8s/K8sClientService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java similarity index 100% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/k8s/K8sClientService.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/k8s/K8sManager.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java similarity index 100% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/k8s/K8sManager.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java similarity index 100% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java similarity index 100% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckService.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java similarity index 100% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckServiceImpl.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/k8s/K8sManagerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java similarity index 100% rename from dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/k8s/K8sManagerTest.java rename to dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceTest.java similarity index 100% rename from dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckServiceTest.java rename to dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceTest.java diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 9e7d5d6016..67c932e96a 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -90,10 +90,5 @@ provided - - io.fabric8 - kubernetes-client - provided - diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml index c689349002..ee865769fa 100644 --- a/dolphinscheduler-worker/pom.xml +++ b/dolphinscheduler-worker/pom.xml @@ -34,6 +34,12 @@ org.apache.dolphinscheduler dolphinscheduler-service + + + org.springframework.boot + spring-boot-starter-quartz + + org.apache.dolphinscheduler diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 009ab3996c..05d7718bd2 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -54,11 +54,21 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; @SpringBootApplication @EnableTransactionManagement -@ComponentScan("org.apache.dolphinscheduler") +@ComponentScan(basePackages = "org.apache.dolphinscheduler", + excludeFilters = { + @ComponentScan.Filter(type = FilterType.REGEX, pattern = { + "org.apache.dolphinscheduler.service.process.*", + // todo: split the quartz into a single module + "org.apache.dolphinscheduler.service.quartz.*", + "org.apache.dolphinscheduler.service.queue.*", + }) + } +) public class WorkerServer implements IStoppable { /** @@ -102,6 +112,8 @@ public class WorkerServer implements IStoppable { @Autowired private WorkerRegistryClient workerRegistryClient; + // todo: Can we just load the task spi, and don't install into mysql? + // we don't need to rely the dao module in worker. @Autowired private TaskPluginManager taskPluginManager; diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java index 45716331a4..33a4d8c21d 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java @@ -17,32 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import org.apache.dolphinscheduler.dao.AlertDao; -import org.apache.dolphinscheduler.dao.datasource.SpringConnectionFactory; -import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; -import org.apache.dolphinscheduler.dao.mapper.AlertMapper; -import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.CommandMapper; -import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; -import org.apache.dolphinscheduler.dao.mapper.DqComparisonTypeMapper; -import org.apache.dolphinscheduler.dao.mapper.DqExecuteResultMapper; -import org.apache.dolphinscheduler.dao.mapper.DqRuleExecuteSqlMapper; -import org.apache.dolphinscheduler.dao.mapper.DqRuleInputEntryMapper; -import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper; -import org.apache.dolphinscheduler.dao.mapper.DqTaskStatisticsValueMapper; -import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; -import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; -import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; -import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; -import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.mockito.Mockito; @@ -55,139 +29,9 @@ import org.springframework.context.annotation.Configuration; @Configuration public class TaskCallbackServiceTestConfig { - @Bean - public AlertDao alertDao() { - return new AlertDao(); - } - - @Bean - public AlertMapper alertMapper() { - return Mockito.mock(AlertMapper.class); - } - @Bean public ProcessService processService() { return Mockito.mock(ProcessService.class); } - @Bean - public UserMapper userMapper() { - return Mockito.mock(UserMapper.class); - } - - @Bean - public ProcessDefinitionMapper processDefineMapper() { - return Mockito.mock(ProcessDefinitionMapper.class); - } - - @Bean - public ProcessInstanceMapper processInstanceMapper() { - return Mockito.mock(ProcessInstanceMapper.class); - } - - @Bean - public DataSourceMapper dataSourceMapper() { - return Mockito.mock(DataSourceMapper.class); - } - - @Bean - public ProcessInstanceMapMapper processInstanceMapMapper() { - return Mockito.mock(ProcessInstanceMapMapper.class); - } - - @Bean - public TaskInstanceMapper taskInstanceMapper() { - return Mockito.mock(TaskInstanceMapper.class); - } - - @Bean - public CommandMapper commandMapper() { - return Mockito.mock(CommandMapper.class); - } - - @Bean - public ScheduleMapper scheduleMapper() { - return Mockito.mock(ScheduleMapper.class); - } - - @Bean - public UdfFuncMapper udfFuncMapper() { - return Mockito.mock(UdfFuncMapper.class); - } - - @Bean - public ResourceMapper resourceMapper() { - return Mockito.mock(ResourceMapper.class); - } - - @Bean - public ResourceUserMapper resourceUserMapper() { - return Mockito.mock(ResourceUserMapper.class); - } - - @Bean - public ErrorCommandMapper errorCommandMapper() { - return Mockito.mock(ErrorCommandMapper.class); - } - - @Bean - public TenantMapper tenantMapper() { - return Mockito.mock(TenantMapper.class); - } - - @Bean - public ProjectMapper projectMapper() { - return Mockito.mock(ProjectMapper.class); - } - - @Bean - public AlertPluginInstanceMapper alertPluginInstanceMapper() { - return Mockito.mock(AlertPluginInstanceMapper.class); - } - - @Bean - public AlertGroupMapper alertGroupMapper() { - return Mockito.mock(AlertGroupMapper.class); - } - - @Bean - public PluginDefineMapper pluginDefineMapper() { - return Mockito.mock(PluginDefineMapper.class); - } - - @Bean - public DqExecuteResultMapper dqExecuteResultMapper() { - return Mockito.mock(DqExecuteResultMapper.class); - } - - @Bean - public DqRuleMapper dqRuleMapper() { - return Mockito.mock(DqRuleMapper.class); - } - - @Bean - public DqRuleInputEntryMapper dqRuleInputEntryMapper() { - return Mockito.mock(DqRuleInputEntryMapper.class); - } - - @Bean - public DqRuleExecuteSqlMapper dqRuleExecuteSqlMapper() { - return Mockito.mock(DqRuleExecuteSqlMapper.class); - } - - @Bean - public DqComparisonTypeMapper dqComparisonTypeMapper() { - return Mockito.mock(DqComparisonTypeMapper.class); - } - - @Bean - public DqTaskStatisticsValueMapper dqTaskStatisticsValueMapper() { - return Mockito.mock(DqTaskStatisticsValueMapper.class); - } - - @Bean - public SpringConnectionFactory springConnectionFactory() { - return Mockito.mock(SpringConnectionFactory.class); - } - }