diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 477b747d44..82142768ce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AccessTokenService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index b9473ecadb..6453de0276 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AlertGroupService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java index bfe340f38d..a2ac7e0410 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java @@ -30,9 +30,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; - +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java index 8fedd7b3d1..c8dcdff039 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ClusterService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java index 4249412810..04203651fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java @@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.api.service.DqExecuteResultService; import org.apache.dolphinscheduler.api.service.DqRuleService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index c4ea3d9271..b344073664 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -36,11 +36,11 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.service.utils.CommonUtils; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java index 6ef432a502..72ea72d1b0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.EnvironmentService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java index e6e77e4904..06a5adfdeb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.K8sNamespaceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.K8sNamespace; import org.apache.dolphinscheduler.dao.entity.User; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index ecdc4b89f3..55c66a6adb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -45,7 +45,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum; import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.User; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 3fc3bb7e8c..b36ba8abd5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.api.service.ProcessInstanceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.slf4j.Logger; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index a53bdcc550..76997f75dc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -32,7 +32,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java index 035bfcc67e..1dd6324a49 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java @@ -42,7 +42,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java index e4d6bc5cec..32734e811e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java @@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java index a19aaed2c3..d00f63cb20 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java @@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 0e4f4b7334..ea2c59c418 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -49,7 +49,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.spi.enums.ResourceType; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index cc01db15a7..2f87a1bf00 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -36,9 +36,8 @@ import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; - +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java index b6060d78c4..301fe085fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java @@ -36,7 +36,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 1ee0f30f78..9b7b300d4f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.api.service.TaskInstanceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index 73a9ee55f1..8c99eb8cfd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 30bb447b4c..1e427b0e23 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.UsersService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java index b92d25d4aa..f887f064b7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java @@ -28,7 +28,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java index 69e1a57d0f..7314a38eed 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java @@ -27,7 +27,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.WorkerGroupService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java index 88309cdb10..fd9a249328 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.api.k8s; -import org.apache.dolphinscheduler.common.utils.ClusterConfUtils; +import org.apache.dolphinscheduler.api.utils.ClusterConfUtils; import org.apache.dolphinscheduler.dao.entity.Cluster; import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; @@ -128,4 +128,4 @@ public class K8sManager { } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index c1881e2c24..b7d4e01d5e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -21,10 +21,10 @@ import org.apache.dolphinscheduler.api.dto.ClusterDto; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.k8s.K8sManager; import org.apache.dolphinscheduler.api.service.ClusterService; +import org.apache.dolphinscheduler.api.utils.ClusterConfUtils; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.ClusterConfUtils; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.dao.entity.Cluster; @@ -384,4 +384,3 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic } } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index c1b13f7d32..e3d82612f4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -69,7 +69,6 @@ import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; @@ -109,6 +108,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; @@ -1004,7 +1004,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // check process instances is already running List processInstances = processInstanceService - .queryByProcessDefineCodeAndStatus(processDefinition.getCode(), Constants.NOT_TERMINATED_STATES); + .queryByProcessDefineCodeAndStatus(processDefinition.getCode(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); if (CollectionUtils.isNotEmpty(processInstances)) { throw new ServiceException(Status.DELETE_PROCESS_DEFINITION_EXECUTING_FAIL, processInstances.size()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index e41543ff4e..4f146a8c2f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -37,11 +37,10 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -68,6 +67,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.service.expand.CuringParamsService; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index a82173cad2..abb8135ad4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -41,7 +41,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -57,6 +56,7 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.beanutils.BeanMap; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 5d866966d4..169b0a6348 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -41,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -304,7 +304,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService } private List getProcessInstancesByTenant(Tenant tenant) { - return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), Constants.NOT_TERMINATED_STATES); + return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 86f2d7a3dd..7ed97daada 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.EncryptionUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.AlertGroup; @@ -56,7 +55,7 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; - +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index a6d46efd39..d95a2017f3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -322,7 +322,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro return result; } List processInstances = processInstanceMapper - .queryByWorkerGroupNameAndStatus(workerGroup.getName(), Constants.NOT_TERMINATED_STATES); + .queryByWorkerGroupNameAndStatus(workerGroup.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); if (CollectionUtils.isNotEmpty(processInstances)) { List processInstanceIds = processInstances.stream().map(ProcessInstance::getId).collect(Collectors.toList()); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClusterConfUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ClusterConfUtils.java similarity index 93% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClusterConfUtils.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ClusterConfUtils.java index 922faefa10..4c3f851fc3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ClusterConfUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ClusterConfUtils.java @@ -15,8 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.api.utils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 7e15784e9c..48d085685d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -134,7 +134,7 @@ public class WorkerGroupControllerTest extends AbstractControllerTest { workerGroup.setId(12); workerGroup.setName("测试"); Mockito.when(workerGroupMapper.selectById(12)).thenReturn(workerGroup); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", Constants.NOT_TERMINATED_STATES)) + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) .thenReturn(null); Mockito.when(workerGroupMapper.deleteById(12)).thenReturn(1); Mockito.when(processInstanceMapper.updateProcessInstanceByWorkerGroupName("测试", "")).thenReturn(1); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java index eefc6ce254..be4c433e2e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java @@ -22,9 +22,8 @@ import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.dao.entity.User; - +import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; import java.util.HashMap; import java.util.Map; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index c2de44783a..e648977f61 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -61,6 +60,7 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.expand.CuringParamsService; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index e170256002..58c1d23d9d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.Resource; @@ -42,6 +41,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections.CollectionUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 2c0419474b..30142be13d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -203,7 +203,7 @@ public class TenantServiceTest { Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.TENANT, null, 0, baseServiceLogger)).thenReturn(true); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)) + Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) .thenReturn(getInstanceList()); Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList()); Mockito.when(userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 992137e1d7..73002855a0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.EncryptionUtils; import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.Project; @@ -49,6 +48,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections.CollectionUtils; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java index decd20bb89..f25c82f19a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java @@ -89,7 +89,7 @@ public class WorkerGroupServiceTest { user.setUserType(UserType.ADMIN_USER); WorkerGroup wg2 = getWorkerGroup(2); Mockito.when(workerGroupMapper.selectById(2)).thenReturn(wg2); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList()); + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList()); Map result = workerGroupService.deleteWorkerGroupById(user, 1); Assert.assertEquals(Status.DELETE_WORKER_GROUP_NOT_EXIST.getCode(), ((Status) result.get(Constants.STATUS)).getCode()); result = workerGroupService.deleteWorkerGroupById(user, 2); @@ -97,7 +97,7 @@ public class WorkerGroupServiceTest { // correct WorkerGroup wg3 = getWorkerGroup(3); Mockito.when(workerGroupMapper.selectById(3)).thenReturn(wg3); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), Constants.NOT_TERMINATED_STATES)).thenReturn(new ArrayList<>()); + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(new ArrayList<>()); result = workerGroupService.deleteWorkerGroupById(user, 3); Assert.assertEquals(Status.SUCCESS.getMsg(), result.get(Constants.MSG)); } diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 00806f5298..4bd4034ead 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -42,13 +42,8 @@ - org.apache.dolphinscheduler - dolphinscheduler-spi - - - - org.apache.dolphinscheduler - dolphinscheduler-task-api + commons-io + commons-io @@ -99,208 +94,6 @@ org.apache.commons commons-lang3 - - org.apache.hadoop - hadoop-common - provided - - - org.slf4j - slf4j-log4j12 - - - jdk.tools - jdk.tools - - - javax.servlet - servlet-api - - - javax.servlet - servlet-api - - - log4j - log4j - - - org.apache.curator - curator-client - - - - commons-configuration - commons-configuration - - - io.grpc - grpc-protobuf - - - io.netty - netty - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - - com.google.protobuf - jackson-mapper-asl - - - com.google.code.gson - gson - - - xmlenc - xmlenc - - - commons-net - commons-net - - - org.apache.avro - avro - - - org.apache.zookeeper - zookeeper - - - javax.servlet.jsp - jsp-api - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - org.mortbay.jetty - jetty - - - - - org.apache.hbase.thirdparty - hbase-noop-htrace - - - org.apache.hadoop - hadoop-client - provided - - - org.slf4j - slf4j-log4j12 - - - javax.servlet - servlet-api - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - - org.fusesource.leveldbjni - leveldbjni-all - - - org.apache.zookeeper - zookeeper - - - org.apache.hadoop - hadoop-mapreduce-client-shuffle - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-core - - - javax.xml.bind - jaxb-api - - - log4j - log4j - - - - - - org.apache.hadoop - hadoop-hdfs - provided - - - javax.servlet - servlet-api - - - io.netty - netty - - - com.google.protobuf - protobuf-java - - - xmlenc - xmlenc - - - io.netty - netty-all - - - org.fusesource.leveldbjni - leveldbjni-all - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - log4j - log4j - - - - - - com.amazonaws - aws-java-sdk-s3 - provided - com.aliyun.oss diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 81a0272e4d..561d60925f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.common; -import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; - import org.apache.commons.lang3.SystemUtils; import java.time.Duration; @@ -642,23 +639,6 @@ public final class Constants { */ public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s"; - public static final int[] NOT_TERMINATED_STATES = new int[]{ - WorkflowExecutionStatus.SUBMITTED_SUCCESS.getCode(), - TaskExecutionStatus.DISPATCH.getCode(), - WorkflowExecutionStatus.RUNNING_EXECUTION.getCode(), - WorkflowExecutionStatus.DELAY_EXECUTION.getCode(), - WorkflowExecutionStatus.READY_PAUSE.getCode(), - WorkflowExecutionStatus.READY_STOP.getCode(), - TaskExecutionStatus.NEED_FAULT_TOLERANCE.getCode(), - }; - - public static final int[] RUNNING_PROCESS_STATE = new int[]{ - TaskExecutionStatus.RUNNING_EXECUTION.getCode(), - TaskExecutionStatus.SUBMITTED_SUCCESS.getCode(), - TaskExecutionStatus.DISPATCH.getCode(), - WorkflowExecutionStatus.SERIAL_WAIT.getCode() - }; - /** * status */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index b94e34e510..3fd81942da 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -17,6 +17,28 @@ package org.apache.dolphinscheduler.common.utils; +import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; +import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; +import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; +import static java.nio.charset.StandardCharsets.UTF_8; + +import org.apache.dolphinscheduler.common.Constants; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; @@ -33,26 +55,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.type.CollectionType; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.TimeZone; - -import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; -import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; -import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; -import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.base.Strings; /** * json utils @@ -62,7 +65,7 @@ public class JSONUtils { private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); static { - logger.info("init timezone: {}",TimeZone.getDefault()); + logger.info("init timezone: {}", TimeZone.getDefault()); } /** @@ -129,7 +132,7 @@ public class JSONUtils { * classOfT */ public static @Nullable T parseObject(String json, Class clazz) { - if (StringUtils.isEmpty(json)) { + if (Strings.isNullOrEmpty(json)) { return null; } @@ -166,7 +169,7 @@ public class JSONUtils { * @return list */ public static List toList(String json, Class clazz) { - if (StringUtils.isEmpty(json)) { + if (Strings.isNullOrEmpty(json)) { return Collections.emptyList(); } @@ -188,7 +191,7 @@ public class JSONUtils { */ public static boolean checkJsonValid(String json) { - if (StringUtils.isEmpty(json)) { + if (Strings.isNullOrEmpty(json)) { return false; } @@ -229,7 +232,8 @@ public class JSONUtils { * @return json to map */ public static Map toMap(String json) { - return parseObject(json, new TypeReference>() {}); + return parseObject(json, new TypeReference>() { + }); } /** @@ -243,7 +247,7 @@ public class JSONUtils { * @return to map */ public static Map toMap(String json, Class classK, Class classV) { - if (StringUtils.isEmpty(json)) { + if (Strings.isNullOrEmpty(json)) { return Collections.emptyMap(); } @@ -285,7 +289,7 @@ public class JSONUtils { * @return return parse object */ public static T parseObject(String json, TypeReference type) { - if (StringUtils.isEmpty(json)) { + if (Strings.isNullOrEmpty(json)) { return null; } @@ -319,7 +323,7 @@ public class JSONUtils { * @param object type * @return byte array */ - public static byte[] toJsonByteArray(T obj) { + public static byte[] toJsonByteArray(T obj) { if (obj == null) { return null; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index b5203a9245..8faba94638 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.shell.ShellExecutor; import oshi.SystemInfo; @@ -417,12 +418,16 @@ public class OSUtils { * @return result of sudo execute command */ public static String getSudoCmd(String tenantCode, String command) { - if (!CommonUtils.isSudoEnable() || StringUtils.isEmpty(tenantCode)) { + if (!isSudoEnable() || StringUtils.isEmpty(tenantCode)) { return command; } return String.format("sudo -u %s %s", tenantCode, command); } + public static boolean isSudoEnable() { + return PropertyUtils.getBoolean(Constants.SUDO_ENABLE, true); + } + /** * Execute the corresponding command of Linux or Windows * diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index bd8c259eb3..5c6ff3b0ed 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.spi.enums.ResUploadType; +import org.apache.dolphinscheduler.common.enums.ResUploadType; import java.io.IOException; import java.io.InputStream; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java index df8050af41..b8c1cdac65 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.common.utils; +import static org.junit.Assert.assertNotNull; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.spi.enums.ResUploadType; + import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertNotNull; - public class PropertyUtilsTest { @Test @@ -32,7 +32,7 @@ public class PropertyUtilsTest { } @Test - public void getResUploadStartupState(){ + public void getResUploadStartupState() { Assert.assertFalse(PropertyUtils.getResUploadStartupState()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java deleted file mode 100644 index 66536baa65..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtilsTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils.placeholder; - -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholderUtils; - -import java.util.Date; -import java.util.TimeZone; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TimePlaceholderUtilsTest { - - private Date date; - - @BeforeClass - public static void setup() { - TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai")); - } - - @Before - public void init() { - date = DateUtils.parse("20170101010101", "yyyyMMddHHmmss", null); - } - - @Test - public void replacePlaceholdersT() { - Assert.assertEquals("2017test12017:***2016-12-31,20170102,20170130,20161227,20161231", TimePlaceholderUtils - .replacePlaceholders("$[yyyy]test1$[yyyy:***]$[yyyy-MM-dd-1],$[month_begin(yyyyMMdd, 1)],$[month_end(yyyyMMdd, -1)],$[week_begin(yyyyMMdd, 1)],$[week_end(yyyyMMdd, -1)]", - date, true)); - - Assert.assertEquals("1483200061,1483290061,1485709261,1482771661,1483113600,1483203661", TimePlaceholderUtils.replacePlaceholders("$[timestamp(yyyyMMdd00mmss)]," - + "$[timestamp(month_begin(yyyyMMddHHmmss, 1))]," - + "$[timestamp(month_end(yyyyMMddHHmmss, -1))]," - + "$[timestamp(week_begin(yyyyMMddHHmmss, 1))]," - + "$[timestamp(week_end(yyyyMMdd000000, -1))]," - + "$[timestamp(yyyyMMddHHmmss)]", - date, true)); - } - - @Test - public void calcMinutesT() { - Assert.assertEquals("Sun Jan 01 01:01:01 CST 2017=yyyy", TimePlaceholderUtils.calcMinutes("yyyy", date).toString()); - Assert.assertEquals("Sun Jan 08 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+7*1", date).toString()); - Assert.assertEquals("Sun Dec 25 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-7*1", date).toString()); - Assert.assertEquals("Mon Jan 02 01:01:01 CST 2017=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd+1", date).toString()); - Assert.assertEquals("Sat Dec 31 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMinutes("yyyyMMdd-1", date).toString()); - Assert.assertEquals("Sun Jan 01 02:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH+1/24", date).toString()); - Assert.assertEquals("Sun Jan 01 00:01:01 CST 2017=yyyyMMddHH", TimePlaceholderUtils.calcMinutes("yyyyMMddHH-1/24", date).toString()); - } - - @Test - public void calcMonthsT() { - Assert.assertEquals("Mon Jan 01 01:01:01 CST 2018=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,12*1)", date).toString()); - Assert.assertEquals("Fri Jan 01 01:01:01 CST 2016=yyyyMMdd", TimePlaceholderUtils.calcMonths("add_months(yyyyMMdd,-12*1)", date).toString()); - } - - @Test - public void testGetPlaceHolderTime() { - - Assert.assertEquals("20170101", TimePlaceholderUtils.getPlaceHolderTime("yyyyMMdd", date)); - } - -} diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 1090096f65..e5620d1588 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -15,7 +15,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 @@ -27,6 +26,18 @@ dolphinscheduler-dao ${project.artifactId} + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + @@ -34,6 +45,11 @@ dolphinscheduler-common + + org.apache.dolphinscheduler + dolphinscheduler-task-api + + org.springframework.boot spring-boot-starter @@ -43,8 +59,8 @@ spring-boot-starter-tomcat - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -98,18 +114,6 @@ - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessData.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessData.java deleted file mode 100644 index 005f8cb51d..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessData.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.entity; - -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; - -import java.util.List; - -/** - * definition json data structure - */ -public class ProcessData { - /** - * task list - */ - private List tasks; - - /** - * global parameters - */ - private List globalParams; - - - private int timeout; - - private int tenantId; - - - public ProcessData() { - } - - /** - * - * @param tasks tasks - * @param globalParams globalParams - */ - public ProcessData(List tasks, List globalParams) { - this.tasks = tasks; - this.globalParams = globalParams; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - ProcessData that = (ProcessData) o; - - return CollectionUtils.equalLists(tasks, that.tasks) && - CollectionUtils.equalLists(globalParams, that.globalParams); - } - - public List getTasks() { - return tasks; - } - - public void setTasks(List tasks) { - this.tasks = tasks; - } - - public List getGlobalParams() { - return globalParams; - } - - public void setGlobalParams(List globalParams) { - this.globalParams = globalParams; - } - - public int getTimeout() { - return timeout; - } - - public void setTimeout(int timeout) { - this.timeout = timeout; - } - - public int getTenantId() { - return tenantId; - } - - public void setTenantId(int tenantId) { - this.tenantId = tenantId; - } - - @Override - public String toString() { - return "ProcessData{" + - "tasks=" + tasks + - ", globalParams=" + globalParams + - ", timeout=" + timeout + - ", tenantId=" + tenantId + - '}'; - } -} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java index 0759bdb6c1..73af3517db 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java @@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*; import static org.apache.dolphinscheduler.spi.utils.Constants.RESOURCE_STORAGE_TYPE; +import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.spi.enums.ResUploadType; import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.hadoop.conf.Configuration; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/utils/CommonUtil.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/utils/CommonUtil.java index ebb6a26279..97ccbb84d0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/utils/CommonUtil.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/utils/CommonUtil.java @@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.utils; import static org.apache.dolphinscheduler.spi.utils.Constants.JAVA_SECURITY_KRB5_CONF; +import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.spi.enums.ResUploadType; import org.apache.dolphinscheduler.spi.utils.Constants; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -30,10 +30,10 @@ import org.apache.hadoop.security.UserGroupInformation; import java.io.IOException; import java.util.Objects; -public class CommonUtil { +import lombok.experimental.UtilityClass; - private CommonUtil() { - } +@UtilityClass +public class CommonUtil { public static boolean getKerberosStartupState() { String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java index 9e859cd2e7..a54fc2cbac 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java @@ -22,7 +22,6 @@ import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.WorkflowStateEventChangeCommand; @@ -31,6 +30,7 @@ import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java index 22faf7f05a..ce31b1a018 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java @@ -21,13 +21,13 @@ import com.google.common.base.Preconditions; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskEventChangeCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java index 775aded34a..8f66b612db 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java @@ -18,14 +18,13 @@ package org.apache.dolphinscheduler.server.master.processor; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java index a5d404ec65..5ed1a9262a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java @@ -18,14 +18,13 @@ package org.apache.dolphinscheduler.server.master.processor; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskRejectCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java index b30c1dfa37..75cf8a9d0b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java @@ -20,12 +20,12 @@ package org.apache.dolphinscheduler.server.master.processor.queue; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java index 3c05671a10..cad01a325b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java @@ -18,11 +18,10 @@ package org.apache.dolphinscheduler.server.master.processor.queue; import org.apache.dolphinscheduler.common.enums.TaskEventType; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.master.event.TaskEventHandleError; import org.apache.dolphinscheduler.server.master.event.TaskEventHandleException; import org.apache.dolphinscheduler.server.master.event.TaskEventHandler; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java index 14b0252d7c..e1c87de8f3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java @@ -20,10 +20,9 @@ package org.apache.dolphinscheduler.server.master.runner; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java index 351ebe7c17..89bc42e61f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.SlotCheckState; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.Command; @@ -41,7 +40,7 @@ import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.process.ProcessService; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java index 8a4b195a5b..28d31f060a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java index edc815d721..6db170d277 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; @@ -58,6 +57,7 @@ import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections.CollectionUtils; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java index ba845b7d15..23d4ca8521 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java @@ -17,11 +17,10 @@ package org.apache.dolphinscheduler.server.master.runner; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.process.ProcessService; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import javax.annotation.PostConstruct; import org.slf4j.Logger; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java index 37e6143406..1842885348 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java @@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.master.event.WorkflowEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandleError; import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandleException; import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandler; import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; import org.apache.dolphinscheduler.server.master.event.WorkflowEventType; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index 14b88b773b..962ed187d6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -40,13 +40,10 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.Environment; @@ -59,7 +56,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -83,9 +79,12 @@ import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; import org.apache.dolphinscheduler.service.cron.CronUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.expand.CuringParamsService; +import org.apache.dolphinscheduler.service.model.TaskNode; +import org.apache.dolphinscheduler.service.process.ProcessDag; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; - +import org.apache.dolphinscheduler.service.utils.DagHelper; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java index 84e784e6c9..388637a567 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java @@ -21,7 +21,6 @@ import com.google.common.base.Strings; import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java index 66a5b493a1..21a99434f5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java @@ -36,9 +36,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConst import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TARGET_DATASOURCE_ID; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DqComparisonType; @@ -77,7 +75,9 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; import org.apache.dolphinscheduler.service.task.TaskPluginManager; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java index cf50ceec13..9fdc150e0a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master.service; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -39,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactor import org.apache.dolphinscheduler.service.log.LogClient; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.registry.RegistryClient; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java index c7ea9c13db..463f09e7ad 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -38,6 +37,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactor import org.apache.dolphinscheduler.service.log.LogClient; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.registry.RegistryClient; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils; import org.apache.commons.collections4.CollectionUtils; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java index 567f7e99ec..6a2b889095 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java @@ -21,7 +21,6 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYP import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -37,6 +36,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.task.BlockingTaskProcessor; import org.apache.dolphinscheduler.server.master.runner.task.TaskAction; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import java.time.Duration; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java index 7c77e6777f..40d2b88333 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -33,6 +32,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParamete import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import java.time.Duration; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java index c8a3a3443f..157174cf7e 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Before; import org.junit.Test; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 904896bd9f..948c9d24f3 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.master; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import java.util.Calendar; @@ -62,4 +62,4 @@ public class ParamsTest { logger.info("complement data : {}",command); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java index cb2eb7afe3..4c02bbf95b 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; @@ -33,6 +32,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.task.SubTaskProcessor; import org.apache.dolphinscheduler.server.master.runner.task.TaskAction; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import java.time.Duration; diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index c3a4be1555..edc8dd50be 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -17,23 +17,39 @@ --> + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-remote dolphinscheduler-remote + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + org.apache.dolphinscheduler dolphinscheduler-common + + org.apache.dolphinscheduler + dolphinscheduler-task-api + io.netty netty-all @@ -73,16 +89,4 @@ - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java index edbaea13ce..ccc4af829d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java @@ -17,17 +17,18 @@ package org.apache.dolphinscheduler.remote.utils; -import lombok.NonNull; -import static org.apache.dolphinscheduler.common.Constants.COLON; - import java.io.Serializable; import java.util.Objects; +import lombok.NonNull; + /** * server address */ public class Host implements Serializable { + private static final String COLON = ":"; + public static final Host EMPTY = new Host(); /** diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 424b25324a..0b6e66507f 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -91,7 +91,7 @@ org.apache.hadoop hadoop-common - test + provided org.slf4j @@ -178,6 +178,10 @@ com.sun.jersey jersey-core + + org.mortbay.jetty + jetty + @@ -187,7 +191,7 @@ org.apache.hadoop hadoop-client - test + provided org.slf4j @@ -236,5 +240,55 @@ + + + org.apache.hadoop + hadoop-hdfs + provided + + + javax.servlet + servlet-api + + + io.netty + netty + + + com.google.protobuf + protobuf-java + + + xmlenc + xmlenc + + + io.netty + netty-all + + + org.fusesource.leveldbjni + leveldbjni-all + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + log4j + log4j + + + + + + com.amazonaws + aws-java-sdk-s3 + provided + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java index 2ff4e9e7a6..859fcf280a 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParams.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -32,6 +31,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.Date; @@ -42,11 +42,11 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import lombok.NonNull; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import lombok.NonNull; - @Component public class CuringGlobalParams implements CuringParamsService { @@ -79,7 +79,9 @@ public class CuringGlobalParams implements CuringParamsService { * @return */ @Override - public String curingGlobalParams(Integer processInstanceId, Map globalParamMap, List globalParamList, CommandType commandType, Date scheduleTime, String timezone) { + public String curingGlobalParams(Integer processInstanceId, Map globalParamMap, + List globalParamList, CommandType commandType, Date scheduleTime, + String timezone) { if (globalParamList == null || globalParamList.isEmpty()) { return null; } @@ -88,9 +90,8 @@ public class CuringGlobalParams implements CuringParamsService { globalMap.putAll(globalParamMap); } Map allParamMap = new HashMap<>(); - //If it is a complement, a complement time needs to be passed in, according to the task type - Map timeParams = BusinessTimeUtils. - getBusinessTime(commandType, scheduleTime, timezone); + // If it is a complement, a complement time needs to be passed in, according to the task type + Map timeParams = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timezone); if (timeParams != null) { allParamMap.putAll(timeParams); @@ -130,9 +131,11 @@ public class CuringGlobalParams implements CuringParamsService { * @return */ @Override - public Map paramParsingPreparation(@NonNull TaskInstance taskInstance, @NonNull AbstractParameters parameters, @NonNull ProcessInstance processInstance) { + public Map paramParsingPreparation(@NonNull TaskInstance taskInstance, + @NonNull AbstractParameters parameters, + @NonNull ProcessInstance processInstance) { // assign value to definedParams here - Map globalParamsMap = setGlobalParamsMap(processInstance); + Map globalParamsMap = setGlobalParamsMap(processInstance); Map globalParams = ParamUtils.getUserDefParamsMap(globalParamsMap); CommandType commandType = processInstance.getCmdTypeIfComplement(); Date scheduleTime = processInstance.getScheduleTime(); @@ -140,7 +143,7 @@ public class CuringGlobalParams implements CuringParamsService { // combining local and global parameters Map localParams = parameters.getInputLocalParametersMap(); - //stream pass params + // stream pass params parameters.setVarPool(taskInstance.getVarPool()); Map varParams = parameters.getVarPoolMap(); @@ -152,7 +155,7 @@ public class CuringGlobalParams implements CuringParamsService { // of the process instance complement Map cmdParam = JSONUtils.toMap(processInstance.getCommandParam()); String timeZone = cmdParam.get(Constants.SCHEDULE_TIMEZONE); - Map params = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timeZone); + Map params = BusinessTimeUtils.getBusinessTime(commandType, scheduleTime, timeZone); if (globalParamsMap != null) { params.putAll(globalParamsMap); @@ -175,7 +178,8 @@ public class CuringGlobalParams implements CuringParamsService { Map.Entry en = iter.next(); Property property = en.getValue(); - if (StringUtils.isNotEmpty(property.getValue()) && property.getValue().startsWith(Constants.FUNCTION_START_WITH)) { + if (StringUtils.isNotEmpty(property.getValue()) + && property.getValue().startsWith(Constants.FUNCTION_START_WITH)) { /** * local parameter refers to global parameter with the same name * note: the global parameters of the process instance here are solidified parameters, @@ -186,7 +190,7 @@ public class CuringGlobalParams implements CuringParamsService { if (timeFunctionNeedExpand(val)) { val = timeFunctionExtension(taskInstance.getProcessInstanceId(), timeZone, val); } else { - val = convertParameterPlaceholders(val, params); + val = convertParameterPlaceholders(val, params); } property.setValue(val); } @@ -209,7 +213,8 @@ public class CuringGlobalParams implements CuringParamsService { String globalParamsStr = processInstance.getGlobalParams(); if (globalParamsStr != null) { List globalParamsList = JSONUtils.toList(globalParamsStr, Property.class); - globalParamsMap.putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); + globalParamsMap + .putAll(globalParamsList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue))); } return globalParamsMap; } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java index 45fbe6f374..6e3f2a904b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.service.log; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; @@ -36,6 +35,7 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import java.util.List; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessor.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessor.java index 6de75255a7..2e7dac4377 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessor.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.service.log; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.commons.lang3.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java index 227c78b63f..2009aaa25e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/model/TaskNode.java @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.model; +package org.apache.dolphinscheduler.service.model; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; -import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; +import org.apache.dolphinscheduler.common.model.PreviousTaskNode; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; +import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; import org.apache.commons.lang3.StringUtils; @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -139,7 +140,6 @@ public class TaskNode { @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) private String dependence; - @JsonDeserialize(using = JSONUtils.JsonDataDeserializer.class) @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) private String conditionResult; @@ -402,7 +402,8 @@ public class TaskNode { */ public TaskTimeoutParameter getTaskTimeoutParameter() { if (!StringUtils.isEmpty(this.getTimeout())) { - String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); + String formatStr = + String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name()); return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class); } @@ -430,7 +431,8 @@ public class TaskNode { } public String getTaskParams() { - Map taskParams = JSONUtils.parseObject(this.params, new TypeReference>() {}); + Map taskParams = JSONUtils.parseObject(this.params, new TypeReference>() { + }); if (taskParams == null) { taskParams = new HashMap<>(); @@ -443,7 +445,8 @@ public class TaskNode { } public Map taskParamsToJsonObj(String taskParams) { - Map taskParamsMap = JSONUtils.parseObject(taskParams, new TypeReference>() {}); + Map taskParamsMap = JSONUtils.parseObject(taskParams, new TypeReference>() { + }); if (taskParamsMap == null) { taskParamsMap = new HashMap<>(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ProcessDag.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessDag.java similarity index 94% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ProcessDag.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessDag.java index 357e4ca6b1..c4f42c0e51 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/ProcessDag.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessDag.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.process; +package org.apache.dolphinscheduler.service.process; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.service.model.TaskNode; import java.util.List; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index e04e325454..a18d751205 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.service.process; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.dao.entity.Command; @@ -53,6 +52,7 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.DateInterval; import org.apache.dolphinscheduler.service.exceptions.CronParseException; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.spi.enums.ResourceType; import java.util.Date; diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index d25bd083fe..b96b881b4d 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -44,9 +44,7 @@ import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -113,7 +111,6 @@ import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.dao.utils.DqRuleUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -134,7 +131,9 @@ import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.log.LogClient; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.task.TaskPluginManager; +import org.apache.dolphinscheduler.service.utils.DagHelper; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -333,7 +332,8 @@ public class ProcessServiceImpl implements ProcessService { List runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId( processInstance.getProcessDefinitionCode(), - processInstance.getProcessDefinitionVersion(), Constants.RUNNING_PROCESS_STATE, + processInstance.getProcessDefinitionVersion(), + org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, processInstance.getId()); if (CollectionUtils.isEmpty(runningProcessInstances)) { processInstance.setStateWithDesc(WorkflowExecutionStatus.SUBMITTED_SUCCESS, @@ -344,7 +344,8 @@ public class ProcessServiceImpl implements ProcessService { List runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId( processInstance.getProcessDefinitionCode(), - processInstance.getProcessDefinitionVersion(), Constants.RUNNING_PROCESS_STATE, + processInstance.getProcessDefinitionVersion(), + org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, processInstance.getId()); if (CollectionUtils.isNotEmpty(runningProcessInstances)) { processInstance.setStateWithDesc(WorkflowExecutionStatus.STOP, "stop by serial_discard strategy"); @@ -358,7 +359,8 @@ public class ProcessServiceImpl implements ProcessService { List runningProcessInstances = this.processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId( processInstance.getProcessDefinitionCode(), - processInstance.getProcessDefinitionVersion(), Constants.RUNNING_PROCESS_STATE, + processInstance.getProcessDefinitionVersion(), + org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, processInstance.getId()); for (ProcessInstance info : runningProcessInstances) { info.setCommandType(CommandType.STOP); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java similarity index 95% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java index 7854eaa032..df980f4b9f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.storage; +package org.apache.dolphinscheduler.service.storage; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; @@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType; import java.io.IOException; import java.util.List; - public interface StorageOperate { String RESOURCE_UPLOAD_PATH = PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); @@ -127,7 +126,8 @@ public interface StorageOperate { * @return * @throws IOException */ - boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, boolean overwrite) throws IOException; + boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, + boolean overwrite) throws IOException; /** * download the srcPath to local @@ -138,7 +138,8 @@ public interface StorageOperate { * @param overwrite * @throws IOException */ - void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource, boolean overwrite)throws IOException; + void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource, + boolean overwrite) throws IOException; /** * vim the context of filePath diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/StoreConfiguration.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StoreConfiguration.java similarity index 87% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/StoreConfiguration.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StoreConfiguration.java index 871bd7e104..5f299dedee 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/StoreConfiguration.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StoreConfiguration.java @@ -15,18 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.config; +package org.apache.dolphinscheduler.service.storage; import static org.apache.dolphinscheduler.common.Constants.RESOURCE_STORAGE_TYPE; import static org.apache.dolphinscheduler.common.Constants.STORAGE_HDFS; import static org.apache.dolphinscheduler.common.Constants.STORAGE_OSS; import static org.apache.dolphinscheduler.common.Constants.STORAGE_S3; -import org.apache.dolphinscheduler.common.storage.StorageOperate; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.OssOperator; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.S3Utils; +import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; +import org.apache.dolphinscheduler.service.storage.impl.OssOperator; +import org.apache.dolphinscheduler.service.storage.impl.S3Utils; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java similarity index 98% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java index 9f35607fb3..42f7e64077 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.storage.impl; import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; @@ -25,8 +25,13 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_TYPE_UDF; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.exception.BaseException; -import org.apache.dolphinscheduler.common.storage.StorageOperate; +import org.apache.dolphinscheduler.common.utils.HttpUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.KerberosHttpClient; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.service.storage.StorageOperate; +import org.apache.dolphinscheduler.service.utils.CommonUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.io.IOUtils; @@ -350,15 +355,15 @@ public class HadoopUtils implements Closeable, StorageOperate { * copy hdfs file to local * * @param srcHdfsFilePath source hdfs file path - * + * * @param dstFile destination file - * + * * @param deleteSource delete source - * + * * @param overwrite overwrite - * + * * @return result of copy hdfs file to local - * + * * @throws IOException errors */ public boolean copyHdfsToLocal(String srcHdfsFilePath, String dstFile, boolean deleteSource, diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OssOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java similarity index 98% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OssOperator.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java index b7a78f31f2..73ce1bf546 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OssOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.storage.impl; import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; @@ -26,8 +26,9 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.factory.OssClientFactory; import org.apache.dolphinscheduler.common.model.OssConnection; -import org.apache.dolphinscheduler.common.storage.StorageOperate; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java similarity index 85% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java index 9973308447..c40641ddbc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.storage.impl; import static org.apache.dolphinscheduler.common.Constants.AWS_END_POINT; import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; @@ -27,8 +27,9 @@ import static org.apache.dolphinscheduler.common.Constants.STORAGE_S3; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; @@ -80,7 +81,6 @@ public class S3Utils implements Closeable, StorageOperate { public static final String BUCKET_NAME = PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME); - private AmazonS3 s3Client = null; private S3Utils() { @@ -88,17 +88,20 @@ public class S3Utils implements Closeable, StorageOperate { if (!StringUtils.isEmpty(PropertyUtils.getString(AWS_END_POINT))) { s3Client = AmazonS3ClientBuilder - .standard() - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(PropertyUtils.getString(AWS_END_POINT), Regions.fromName(REGION).getName())) - .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID))) - .build(); + .standard() + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( + PropertyUtils.getString(AWS_END_POINT), Regions.fromName(REGION).getName())) + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID))) + .build(); } else { s3Client = AmazonS3ClientBuilder - .standard() - .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID))) - .withRegion(Regions.fromName(REGION)) - .build(); + .standard() + .withCredentials( + new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID))) + .withRegion(Regions.fromName(REGION)) + .build(); } checkBucketNameExists(BUCKET_NAME); } @@ -108,6 +111,7 @@ public class S3Utils implements Closeable, StorageOperate { * S3Utils single */ private enum S3Singleton { + INSTANCE; private final S3Utils instance; @@ -176,7 +180,8 @@ public class S3Utils implements Closeable, StorageOperate { } @Override - public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource, boolean overwrite) throws IOException { + public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource, + boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); @@ -184,8 +189,9 @@ public class S3Utils implements Closeable, StorageOperate { Files.createDirectories(dstFile.getParentFile().toPath()); } S3Object o = s3Client.getObject(BUCKET_NAME, srcFilePath); - try (S3ObjectInputStream s3is = o.getObjectContent(); - FileOutputStream fos = new FileOutputStream(dstFilePath)) { + try ( + S3ObjectInputStream s3is = o.getObjectContent(); + FileOutputStream fos = new FileOutputStream(dstFilePath)) { byte[] readBuf = new byte[1024]; int readLen; while ((readLen = s3is.read(readBuf)) > 0) { @@ -236,7 +242,8 @@ public class S3Utils implements Closeable, StorageOperate { } @Override - public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { + public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, + boolean overwrite) throws IOException { try { s3Client.putObject(BUCKET_NAME, dstPath, new File(srcFile)); return true; @@ -322,7 +329,6 @@ public class S3Utils implements Closeable, StorageOperate { s3Client.putObject(BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(strPath)); } - /** * xxx untest * download S3 Directory to local @@ -334,10 +340,12 @@ public class S3Utils implements Closeable, StorageOperate { private void downloadDirectory(String tenantCode, String keyPrefix, String srcPath) { TransferManager tm = TransferManagerBuilder.standard().withS3Client(s3Client).build(); try { - MultipleFileDownload download = tm.downloadDirectory(BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(srcPath)); + MultipleFileDownload download = + tm.downloadDirectory(BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(srcPath)); download.waitForCompletion(); } catch (AmazonS3Exception | InterruptedException e) { - logger.error("download the directory failed with the bucketName is {} and the keyPrefix is {}", BUCKET_NAME, tenantCode + FOLDER_SEPARATOR + keyPrefix); + logger.error("download the directory failed with the bucketName is {} and the keyPrefix is {}", BUCKET_NAME, + tenantCode + FOLDER_SEPARATOR + keyPrefix); Thread.currentThread().interrupt(); } finally { tm.shutdownNow(); @@ -350,16 +358,17 @@ public class S3Utils implements Closeable, StorageOperate { } Bucket existsBucket = s3Client.listBuckets() - .stream() - .filter( - bucket -> bucket.getName().equals(bucketName) - ) - .findFirst() - .orElseThrow(() -> { - return new IllegalArgumentException("bucketName: " + bucketName + " is not exists, you need to create them by yourself"); - }); - - logger.info("bucketName: {} has been found, the current regionName is {}", existsBucket.getName(), s3Client.getRegionName()); + .stream() + .filter( + bucket -> bucket.getName().equals(bucketName)) + .findFirst() + .orElseThrow(() -> { + return new IllegalArgumentException( + "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); + }); + + logger.info("bucketName: {} has been found, the current regionName is {}", existsBucket.getName(), + s3Client.getRegionName()); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java similarity index 82% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java index acda412aa3..fc325ce032 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/CommonUtils.java @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.utils; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.spi.enums.ResUploadType; +import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; @@ -36,6 +37,7 @@ import org.slf4j.LoggerFactory; * common utils */ public class CommonUtils { + private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); private static final Base64 BASE64 = new Base64(); @@ -70,13 +72,6 @@ public class CommonUtils { return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true); } - /** - * @return sudo enable - */ - public static boolean isSudoEnable() { - return PropertyUtils.getBoolean(Constants.SUDO_ENABLE, true); - } - /** * if upload resource is HDFS and kerberos startup is true , else false * @@ -85,7 +80,8 @@ public class CommonUtils { public static boolean getKerberosStartupState() { String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); - Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); + Boolean kerberosStartupState = + PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); return resUploadType == ResUploadType.HDFS && kerberosStartupState; } @@ -110,7 +106,8 @@ public class CommonUtils { * @param loginUserKeytabPath loginUserKeytabPath * @throws IOException errors */ - public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException { + public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, + String loginUserKeytabPath) throws IOException { loadKerberosConf(javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath, new Configuration()); } @@ -124,13 +121,18 @@ public class CommonUtils { * @return load kerberos config return true * @throws IOException errors */ - public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath, Configuration configuration) throws IOException { + public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, + String loginUserKeytabPath, Configuration configuration) throws IOException { if (CommonUtils.getKerberosStartupState()) { - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))); + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))); configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)), - StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH))); + UserGroupInformation.loginUserFromKeytab( + StringUtils.defaultIfBlank(loginUserKeytabUsername, + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)), + StringUtils.defaultIfBlank(loginUserKeytabPath, + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH))); return true; } return false; @@ -143,14 +145,15 @@ public class CommonUtils { if (StringUtils.isEmpty(password)) { return StringUtils.EMPTY; } - //if encryption is not turned on, return directly + // if encryption is not turned on, return directly boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false); if (!encryptionEnable) { return password; } // Using Base64 + salt to process password - String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, + Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))); return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); } @@ -163,14 +166,15 @@ public class CommonUtils { return StringUtils.EMPTY; } - //if encryption is not turned on, return directly + // if encryption is not turned on, return directly boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false); if (!encryptionEnable) { return password; } // Using Base64 + salt to process password - String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, + Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8); if (!passwordWithSalt.startsWith(salt)) { logger.warn("There is a password and salt mismatch: {} ", password); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/Constants.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/Constants.java new file mode 100644 index 0000000000..babd4c24dd --- /dev/null +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/Constants.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.service.utils; + +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public final class Constants { + + public static final int[] NOT_TERMINATED_STATES = new int[]{ + WorkflowExecutionStatus.SUBMITTED_SUCCESS.getCode(), + TaskExecutionStatus.DISPATCH.getCode(), + WorkflowExecutionStatus.RUNNING_EXECUTION.getCode(), + WorkflowExecutionStatus.DELAY_EXECUTION.getCode(), + WorkflowExecutionStatus.READY_PAUSE.getCode(), + WorkflowExecutionStatus.READY_STOP.getCode(), + TaskExecutionStatus.NEED_FAULT_TOLERANCE.getCode(), + }; + + public static final int[] RUNNING_PROCESS_STATE = new int[]{ + TaskExecutionStatus.RUNNING_EXECUTION.getCode(), + TaskExecutionStatus.SUBMITTED_SUCCESS.getCode(), + TaskExecutionStatus.DISPATCH.getCode(), + WorkflowExecutionStatus.SERIAL_WAIT.getCode() + }; +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java similarity index 99% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java index e2ddca73d9..07cea3ad85 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/DagHelper.java @@ -15,13 +15,11 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils; +package org.apache.dolphinscheduler.service.utils; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -29,6 +27,9 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; +import org.apache.dolphinscheduler.service.model.TaskNode; +import org.apache.dolphinscheduler.service.process.ProcessDag; +import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.apache.commons.collections.CollectionUtils; @@ -38,10 +39,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Optional; +import java.util.Set; -import org.apache.dolphinscheduler.spi.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/LoggerUtils.java similarity index 88% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/LoggerUtils.java index 231d84877b..23b3c5d33e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/LoggerUtils.java @@ -15,31 +15,23 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.utils; -import lombok.experimental.UtilityClass; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; import java.io.BufferedReader; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; + +import lombok.experimental.UtilityClass; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; /** * logger utils @@ -62,7 +54,8 @@ public class LoggerUtils { // like TaskAppId=TASK-20211107-798_1-4084-15210 String firstSubmitTimeStr = DateUtils.format(firstSubmitTime, Constants.YYYYMMDD, null); return String.format("%s=%s-%s-%s_%s-%s-%s", - TaskConstants.TASK_APPID_LOG_FORMAT, TaskConstants.TASK_LOGGER_INFO_PREFIX, firstSubmitTimeStr, processDefineCode, processDefineVersion, processInstId, taskId); + TaskConstants.TASK_APPID_LOG_FORMAT, TaskConstants.TASK_LOGGER_INFO_PREFIX, firstSubmitTimeStr, + processDefineCode, processDefineVersion, processInstId, taskId); } /** diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java similarity index 51% rename from dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java index ef1b8461cd..4481448a1d 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResUploadType.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java @@ -14,12 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.dolphinscheduler.service.utils; -package org.apache.dolphinscheduler.spi.enums; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.service.model.TaskNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import java.util.List; -/** - * data base types - */ -public enum ResUploadType { - HDFS, S3, OSS, NONE +@Data +@NoArgsConstructor +public class ProcessData { + @EqualsAndHashCode.Include + private List tasks; + + @EqualsAndHashCode.Include + private List globalParams; + + private int timeout; + + private int tenantId; + + public ProcessData(List tasks, List globalParams) { + this.tasks = tasks; + this.globalParams = globalParams; + } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java index e1d6a552a7..466dcde68a 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java @@ -18,15 +18,14 @@ package org.apache.dolphinscheduler.service.utils; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.log.LogClient; +import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -181,7 +180,7 @@ public class ProcessUtils { } } - if (CommonUtils.isSudoEnable() && !pidList.isEmpty()) { + if (OSUtils.isSudoEnable() && !pidList.isEmpty()) { pidList = pidList.subList(1, pidList.size()); } return String.join(" ", pidList).trim(); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java index c62748e27c..3e0e73d595 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.service.log; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; @@ -28,6 +27,7 @@ import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import java.nio.charset.StandardCharsets; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java index ad4b7e82e1..09626756fc 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.service.log; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.junit.After; import org.junit.Before; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index 1cb0794875..426942ebc5 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -83,6 +82,7 @@ import org.apache.dolphinscheduler.service.cron.CronUtilsTest; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.expand.CuringParamsService; +import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.base.FormType; @@ -432,7 +432,7 @@ public class ProcessServiceTest { command6.setCommandType(CommandType.RECOVER_SERIAL_WAIT); command6.setProcessDefinitionVersion(1); Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(11L, 1, - Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists); + org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists); Mockito.when(processInstanceMapper.updateNextProcessIdById(223, 222)).thenReturn(true); Mockito.when(commandMapper.deleteById(6)).thenReturn(1); ProcessInstance processInstance6 = processService.handleCommand(host, command6); @@ -454,7 +454,7 @@ public class ProcessServiceTest { command7.setProcessDefinitionVersion(1); Mockito.when(commandMapper.deleteById(7)).thenReturn(1); Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(11L, 1, - Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null); + org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null); ProcessInstance processInstance8 = processService.handleCommand(host, command7); Assert.assertTrue(processInstance8 != null); @@ -477,7 +477,7 @@ public class ProcessServiceTest { command9.setProcessDefinitionVersion(1); Mockito.when(processInstanceMapper.queryDetailById(225)).thenReturn(processInstance9); Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(12L, 1, - Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists); + org.apache.dolphinscheduler.service.utils.Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists); Mockito.when(processInstanceMapper.updateById(processInstance)).thenReturn(1); Mockito.when(commandMapper.deleteById(9)).thenReturn(1); ProcessInstance processInstance10 = processService.handleCommand(host, command9); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java similarity index 95% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java index ecdc557e58..5bf5c55b83 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java @@ -15,8 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.storage.impl; +import org.apache.dolphinscheduler.common.utils.HttpUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.junit.Assert; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OssOperatorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java similarity index 98% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OssOperatorTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java index 569ef65b74..f9132b2732 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OssOperatorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.storage.impl; import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; @@ -29,6 +29,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import org.apache.dolphinscheduler.service.storage.impl.OssOperator; import org.apache.dolphinscheduler.spi.enums.ResourceType; import java.io.IOException; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java similarity index 93% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java index 8bb962107e..3c93031883 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java @@ -15,7 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.service.utils; + +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; import java.net.InetAddress; import java.net.UnknownHostException; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java similarity index 98% rename from dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java rename to dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java index e03aa63cf3..ca2f68e1ab 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/DagHelperTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/DagHelperTest.java @@ -15,23 +15,22 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.utils; +package org.apache.dolphinscheduler.service.utils; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessData; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo; import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; +import org.apache.dolphinscheduler.service.model.TaskNode; +import org.apache.dolphinscheduler.service.process.ProcessDag; import java.io.IOException; import java.util.ArrayList; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java index 516e5836e0..dba655fb36 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java @@ -20,10 +20,10 @@ package org.apache.dolphinscheduler.service.utils; import static org.mockito.ArgumentMatchers.anyString; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml index ea831b6f41..b65e1d1340 100644 --- a/dolphinscheduler-spi/pom.xml +++ b/dolphinscheduler-spi/pom.xml @@ -15,7 +15,8 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler @@ -25,7 +26,24 @@ dolphinscheduler-spi ${project.artifactId} + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + + + org.apache.dolphinscheduler + dolphinscheduler-common + + com.fasterxml.jackson.core @@ -54,16 +72,4 @@ guava - - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java deleted file mode 100644 index 2b1adffee7..0000000000 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/PropertyUtils.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.spi.utils; - -import static org.apache.dolphinscheduler.spi.utils.Constants.COMMON_PROPERTIES_PATH; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PropertyUtils { - - private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); - - private static final Properties properties = new Properties(); - - private PropertyUtils() { - throw new UnsupportedOperationException("Construct PropertyUtils"); - } - - static { - loadPropertyFile(COMMON_PROPERTIES_PATH); - } - - public static synchronized void loadPropertyFile(String... propertyFiles) { - for (String fileName : propertyFiles) { - try (InputStream fis = PropertyUtils.class.getResourceAsStream(fileName);) { - properties.load(fis); - - } catch (IOException e) { - logger.error(e.getMessage(), e); - System.exit(1); - } - } - - // Override from system properties - System.getProperties().forEach((k, v) -> { - final String key = String.valueOf(k); - logger.info("Overriding property from system property: {}", key); - PropertyUtils.setValue(key, String.valueOf(v)); - }); - } - - /** - * get property value - * - * @param key property name - * @return property value - */ - public static String getString(String key) { - return properties.getProperty(key.trim()); - } - - /** - * get property value with upper case - * - * @param key property name - * @return property value with upper case - */ - public static String getUpperCaseString(String key) { - String val = getString(key); - return StringUtils.isEmpty(val) ? val : val.toUpperCase(); - } - - /** - * get property value - * - * @param key property name - * @param defaultVal default value - * @return property value - */ - public static String getString(String key, String defaultVal) { - String val = getString(key); - return StringUtils.isEmpty(val) ? defaultVal : val; - } - - /** - * get property value - * - * @param key property name - * @return get property int value , if key == null, then return -1 - */ - public static int getInt(String key) { - return getInt(key, -1); - } - - /** - * @param key key - * @param defaultValue default value - * @return property value - */ - public static int getInt(String key, int defaultValue) { - String value = getString(key); - if (StringUtils.isEmpty(value)) { - return defaultValue; - } - - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - logger.info(e.getMessage(), e); - } - return defaultValue; - } - - /** - * get property value - * - * @param key property name - * @return property value - */ - public static boolean getBoolean(String key) { - return getBoolean(key, false); - } - - /** - * get property value - * - * @param key property name - * @param defaultValue default value - * @return property value - */ - public static Boolean getBoolean(String key, boolean defaultValue) { - String value = getString(key); - return StringUtils.isEmpty(value) ? defaultValue : Boolean.parseBoolean(value); - } - - /** - * get property long value - * - * @param key key - * @param defaultValue default value - * @return property value - */ - public static long getLong(String key, long defaultValue) { - String value = getString(key); - if (StringUtils.isEmpty(value)) { - return defaultValue; - } - - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - logger.info(e.getMessage(), e); - } - return defaultValue; - } - - /** - * @param key key - * @return property value - */ - public static long getLong(String key) { - return getLong(key, -1); - } - - /** - * set value - * @param key key - * @param value value - */ - public static void setValue(String key, String value) { - properties.setProperty(key, value); - } - -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml index a53870d255..a3d5582ced 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml @@ -15,19 +15,22 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 - jar dolphinscheduler-task-api + jar + + org.apache.dolphinscheduler + dolphinscheduler-common + org.apache.dolphinscheduler dolphinscheduler-spi @@ -73,8 +76,8 @@ provided - slf4j-log4j12 org.slf4j + slf4j-log4j12 org.eclipse.jetty.aggregate @@ -157,128 +160,128 @@ json - log4j-slf4j-impl org.apache.logging.log4j + log4j-slf4j-impl - javax.servlet org.eclipse.jetty.orbit + javax.servlet - servlet-api-2.5 org.mortbay.jetty + servlet-api-2.5 - jasper-runtime tomcat + jasper-runtime - slider-core org.apache.slider + slider-core - hbase-server org.apache.hbase + hbase-server - jersey-client com.sun.jersey + jersey-client - jersey-core com.sun.jersey + jersey-core - jersey-json com.sun.jersey + jersey-json - jersey-server com.sun.jersey + jersey-server - jersey-guice com.sun.jersey.contribs + jersey-guice - hbase-common org.apache.hbase + hbase-common - hbase-hadoop2-compat org.apache.hbase + hbase-hadoop2-compat - hbase-client org.apache.hbase + hbase-client - hbase-hadoop-compat org.apache.hbase + hbase-hadoop-compat - tephra-hbase-compat-1.0 co.cask.tephra + tephra-hbase-compat-1.0 - jaxb-api javax.xml.bind + jaxb-api - hive-llap-client org.apache.hive + hive-llap-client - hive-llap-common org.apache.hive + hive-llap-common - hive-llap-server org.apache.hive + hive-llap-server - tephra-core co.cask.tephra + tephra-core - ant ant + ant - stringtemplate org.antlr + stringtemplate - antlr-runtime org.antlr + antlr-runtime - hive-shims org.apache.hive + hive-shims - jsp-api javax.servlet + jsp-api - log4j-api org.apache.logging.log4j + log4j-api - log4j-core org.apache.logging.log4j + log4j-core - log4j-web org.apache.logging.log4j + log4j-web - jasper-compiler tomcat + jasper-compiler - commons-lang commons-lang + commons-lang diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java index b773b79898..f4395566c7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java @@ -17,14 +17,16 @@ package org.apache.dolphinscheduler.plugin.task.api; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; + +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.utils.AbstractCommandExecutorConstants; import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; + +import org.apache.commons.lang3.SystemUtils; import java.io.BufferedReader; import java.io.File; @@ -43,8 +45,9 @@ import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; +import org.slf4j.Logger; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; /** * abstract command executor @@ -112,7 +115,7 @@ public abstract class AbstractCommandExecutor { // setting up user to run commands List command = new LinkedList<>(); - //init process builder + // init process builder ProcessBuilder processBuilder = new ProcessBuilder(); // setting up a working directory processBuilder.directory(new File(taskRequest.getExecutePath())); @@ -121,7 +124,8 @@ public abstract class AbstractCommandExecutor { // if sudo.enable=true,setting up user to run commands if (OSUtils.isSudoEnable()) { - if (SystemUtils.IS_OS_LINUX && PropertyUtils.getBoolean(AbstractCommandExecutorConstants.TASK_RESOURCE_LIMIT_STATE)) { + if (SystemUtils.IS_OS_LINUX + && PropertyUtils.getBoolean(AbstractCommandExecutorConstants.TASK_RESOURCE_LIMIT_STATE)) { generateCgroupCommand(command); } else { command.add("sudo"); @@ -190,7 +194,7 @@ public abstract class AbstractCommandExecutor { // create command file if not exists createCommandFileIfNotExists(execCommand, commandFilePath); - //build process + // build process buildProcess(commandFilePath); // parse process output @@ -202,7 +206,8 @@ public abstract class AbstractCommandExecutor { // cache processId taskRequest.setProcessId(processId); - boolean updateTaskExecutionContextStatus = TaskExecutionContextCacheManager.updateTaskExecutionContext(taskRequest); + boolean updateTaskExecutionContextStatus = + TaskExecutionContextCacheManager.updateTaskExecutionContext(taskRequest); if (Boolean.FALSE.equals(updateTaskExecutionContextStatus)) { ProcessUtils.kill(taskRequest); result.setExitStatusCode(EXIT_CODE_KILL); @@ -224,12 +229,14 @@ public abstract class AbstractCommandExecutor { result.setExitStatusCode(process.exitValue()); } else { - logger.error("process has failure, the task timeout configuration value is:{}, ready to kill ...", taskRequest.getTaskTimeout()); + logger.error("process has failure, the task timeout configuration value is:{}, ready to kill ...", + taskRequest.getTaskTimeout()); ProcessUtils.kill(taskRequest); result.setExitStatusCode(EXIT_CODE_FAILURE); } - logger.info("process has exited, execute path:{}, processId:{} ,exitStatusCode:{} ,processWaitForStatus:{} ,processExitValue:{}", + logger.info( + "process has exited, execute path:{}, processId:{} ,exitStatusCode:{} ,processWaitForStatus:{} ,processExitValue:{}", taskRequest.getExecutePath(), processId, result.getExitStatusCode(), status, process.exitValue()); return result; @@ -446,7 +453,8 @@ public abstract class AbstractCommandExecutor { /* * when log buffer siz or flush time reach condition , then flush */ - if (logBuffer.size() >= TaskConstants.DEFAULT_LOG_ROWS_NUM || now - lastFlushTime > TaskConstants.DEFAULT_LOG_FLUSH_INTERVAL) { + if (logBuffer.size() >= TaskConstants.DEFAULT_LOG_ROWS_NUM + || now - lastFlushTime > TaskConstants.DEFAULT_LOG_FLUSH_INTERVAL) { lastFlushTime = now; logHandler.accept(logBuffer); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OSUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OSUtils.java index bc9d9c3717..c051055d39 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OSUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OSUtils.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.api.utils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.ShellExecutor; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.io.IOException; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java similarity index 96% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index dae6843112..00b8083e99 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.plugin.task.api.utils; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholderUtils; @@ -36,6 +37,7 @@ import java.util.regex.Pattern; * parameter parse utils */ public class ParameterUtils { + private static final Pattern DATE_PARSE_PATTERN = Pattern.compile("\\$\\[([^\\$\\]]+)]"); private static final Pattern DATE_START_PATTERN = Pattern.compile("^[0-9]"); @@ -60,7 +62,7 @@ public class ParameterUtils { parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); } if (parameterMap != null && null != parameterMap.get(Constants.PARAMETER_DATETIME)) { - //Get current time, schedule execute time + // Get current time, schedule execute time String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME); cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME, null); } else { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java similarity index 87% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java index 472ae8783a..0200ec5d92 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/JSONUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java @@ -15,7 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.plugin.task.api.model; + +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import java.util.ArrayList; import java.util.Date; @@ -25,19 +30,15 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; +import org.junit.Assert; +import org.junit.Test; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; -import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.junit.Assert; -import org.junit.Test; - public class JSONUtilsTest { @Test @@ -47,7 +48,8 @@ public class JSONUtilsTest { property.setDirect(Direct.IN); property.setType(DataType.VARCHAR); property.setValue("sssssss"); - String str = "[{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"},{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}]"; + String str = + "[{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"},{\"prop\":\"ds\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"sssssss\"}]"; JsonNode jsonNode = JSONUtils.toJsonNode(property); ArrayNode arrayNode = JSONUtils.createArrayNode(); @@ -154,7 +156,8 @@ public class JSONUtilsTest { Assert.assertEquals("", JSONUtils.getNodeString("{\"bar\":\"foo\"}", "key")); Assert.assertEquals("foo", JSONUtils.getNodeString("{\"bar\":\"foo\"}", "bar")); Assert.assertEquals("[1,2,3]", JSONUtils.getNodeString("{\"bar\": [1,2,3]}", "bar")); - Assert.assertEquals("{\"1\":\"2\",\"2\":3}", JSONUtils.getNodeString("{\"bar\": {\"1\":\"2\",\"2\":3}}", "bar")); + Assert.assertEquals("{\"1\":\"2\",\"2\":3}", + JSONUtils.getNodeString("{\"bar\": {\"1\":\"2\",\"2\":3}}", "bar")); } @Test @@ -244,27 +247,12 @@ public class JSONUtilsTest { Assert.assertEquals("yellow", node.path(0).path("color").asText()); } - @Test - public void jsonDataDeserializerTest() { - String a = "{\"conditionResult\":\"{\\\"successNode\\\":[\\\"\\\"],\\\"failedNode\\\":[\\\"\\\"]}\"," - + "\"conditionsTask\":false,\"depList\":[],\"dependence\":\"{}\",\"forbidden\":false," - + "\"id\":\"tasks-86823\",\"maxRetryTimes\":1,\"name\":\"shell test\"," - + "\"params\":\"{\\\"resourceList\\\":[],\\\"localParams\\\":[],\\\"rawScript\\\":\\\"echo " - + "'yyc'\\\"}\",\"preTasks\":\"[]\",\"retryInterval\":1,\"runFlag\":\"NORMAL\"," - + "\"taskInstancePriority\":\"HIGHEST\",\"taskTimeoutParameter\":{\"enable\":false,\"interval\":0}," - + "\"timeout\":\"{}\",\"type\":\"SHELL\",\"workerGroup\":\"default\"}"; - - TaskNode taskNode = JSONUtils.parseObject(a, TaskNode.class); - - Assert.assertTrue(true); - } - @Test public void dateToString() { TimeZone timeZone = TimeZone.getTimeZone("UTC"); TimeZone.setDefault(timeZone); JSONUtils.setTimeZone(timeZone); - + String time = "2022-02-22 13:38:24"; Date date = DateUtils.stringToDate(time); String json = JSONUtils.toJsonString(date); @@ -279,12 +267,11 @@ public class JSONUtilsTest { TimeZone timeZone = TimeZone.getTimeZone("UTC"); TimeZone.setDefault(timeZone); JSONUtils.setTimeZone(timeZone); - + String json = "\"2022-02-22 13:38:24\""; Date date = JSONUtils.parseObject(json, Date.class); Assert.assertEquals(DateUtils.stringToDate("2022-02-22 13:38:24"), date); } - } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java similarity index 80% rename from dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 71acc348be..59acc82687 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; - +package org.apache.dolphinscheduler.plugin.task.api.utils; import static org.apache.dolphinscheduler.plugin.task.api.parser.TimePlaceholderUtils.replacePlaceholders; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils; import java.text.ParseException; @@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory; @RunWith(MockitoJUnitRunner.class) public class ParameterUtilsTest { + public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class); /** @@ -54,9 +55,10 @@ public class ParameterUtilsTest { // parameterString、parameterMap is not null String parameterString = "test_parameter"; - Assert.assertEquals(parameterString, ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap)); + Assert.assertEquals(parameterString, + ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap)); - //replace variable ${} form + // replace variable ${} form parameterMap.put("testParameter2", "${testParameter}"); Assert.assertEquals(parameterString, PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true)); @@ -72,14 +74,15 @@ public class ParameterUtilsTest { @Test public void testConvertParameterPlaceholders2() { String parameterString = - "${user} is userName, '$[1]' '$[add_months(yyyyMMdd,12*2)]' '$[add_months(yyyyMMdd,-12*2)]' '$[add_months(yyyyMMdd,3)]' '$[add_months(yyyyMMdd,-4)]' " - + "'$[yyyyMMdd+7*2]' '$[yyyyMMdd-7*2]' '$[yyyyMMdd+3]' '$[0]' '$[yyyyMMdd-3]' '$[HHmmss+2/24]' '$[HHmmss-1/24]' '$[HHmmss+3/24/60]' '$[HHmmss-2/24/60]' '$[3]'"; + "${user} is userName, '$[1]' '$[add_months(yyyyMMdd,12*2)]' '$[add_months(yyyyMMdd,-12*2)]' '$[add_months(yyyyMMdd,3)]' '$[add_months(yyyyMMdd,-4)]' " + + "'$[yyyyMMdd+7*2]' '$[yyyyMMdd-7*2]' '$[yyyyMMdd+3]' '$[0]' '$[yyyyMMdd-3]' '$[HHmmss+2/24]' '$[HHmmss-1/24]' '$[HHmmss+3/24/60]' '$[HHmmss-2/24/60]' '$[3]'"; Map parameterMap = new HashMap<>(); parameterMap.put("user", "Kris"); parameterMap.put(Constants.PARAMETER_DATETIME, "20201201123000"); parameterString = ParameterUtils.convertParameterPlaceholders(parameterString, parameterMap); - Assert.assertEquals("Kris is userName, '$[1]' '20221201' '20181201' '20210301' '20200801' '20201215' '20201117' '20201204' '$[0]' '20201128' '143000' '113000' '123300' '122800' '$[3]'", - parameterString); + Assert.assertEquals( + "Kris is userName, '$[1]' '20221201' '20181201' '20210301' '20200801' '20201215' '20201117' '20201204' '$[0]' '20201128' '143000' '113000' '123300' '122800' '$[3]'", + parameterString); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java index 572cc35537..a9c1255fc8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncHook.java @@ -17,14 +17,10 @@ package org.apache.dolphinscheduler.plugin.task.datasync; -import lombok.Data; -import lombok.SneakyThrows; -import org.apache.commons.beanutils.BeanUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -47,17 +43,27 @@ import software.amazon.awssdk.services.datasync.model.TaskExecutionStatus; import software.amazon.awssdk.services.datasync.model.TaskSchedule; import software.amazon.awssdk.services.datasync.model.TaskStatus; +import org.apache.commons.beanutils.BeanUtils; + import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import lombok.Data; +import lombok.SneakyThrows; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + @Data public class DatasyncHook { - public static TaskExecutionStatus[] doneStatus = {TaskExecutionStatus.ERROR, TaskExecutionStatus.SUCCESS, TaskExecutionStatus.UNKNOWN_TO_SDK_VERSION}; + public static TaskExecutionStatus[] doneStatus = + {TaskExecutionStatus.ERROR, TaskExecutionStatus.SUCCESS, TaskExecutionStatus.UNKNOWN_TO_SDK_VERSION}; public static TaskStatus[] taskFinishFlags = {TaskStatus.UNAVAILABLE, TaskStatus.UNKNOWN_TO_SDK_VERSION}; - protected final Logger logger = LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); + protected final Logger logger = + LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); private DataSyncClient client; private String taskArn; private String taskExecArn; @@ -75,7 +81,8 @@ public class DatasyncHook { final AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials); // create a datasync client - return DataSyncClient.builder().region(Region.of(awsRegion)).credentialsProvider(awsCredentialsProvider).build(); + return DataSyncClient.builder().region(Region.of(awsRegion)).credentialsProvider(awsCredentialsProvider) + .build(); } public Boolean createDatasyncTask(DatasyncParameters parameters) { @@ -109,7 +116,6 @@ public class DatasyncHook { return doubleCheckExecStatus(TaskExecutionStatus.LAUNCHING, doneStatus); } - public Boolean cancelDatasyncTask() { logger.info("cancelTask ......"); CancelTaskExecutionRequest cancel = CancelTaskExecutionRequest.builder().taskExecutionArn(taskExecArn).build(); @@ -135,7 +141,8 @@ public class DatasyncHook { public TaskExecutionStatus queryDatasyncTaskExecStatus() { logger.info("queryDatasyncTaskExecStatus ......"); - DescribeTaskExecutionRequest request = DescribeTaskExecutionRequest.builder().taskExecutionArn(taskExecArn).build(); + DescribeTaskExecutionRequest request = + DescribeTaskExecutionRequest.builder().taskExecutionArn(taskExecArn).build(); DescribeTaskExecutionResponse describe = client.describeTaskExecution(request); if (describe.sdkHttpResponse().isSuccessful()) { @@ -215,10 +222,12 @@ public class DatasyncHook { return false; } - private static void castParamPropertyPackage(DatasyncParameters parameters, CreateTaskRequest.Builder builder){ + private static void castParamPropertyPackage(DatasyncParameters parameters, CreateTaskRequest.Builder builder) { List tags = parameters.getTags(); if (tags != null && tags.size() > 0) { - List collect = tags.stream().map(e -> TagListEntry.builder().key(e.getKey()).value(e.getValue()).build()).collect(Collectors.toList()); + List collect = + tags.stream().map(e -> TagListEntry.builder().key(e.getKey()).value(e.getValue()).build()) + .collect(Collectors.toList()); builder.tags(collect); } DatasyncParameters.Options options = parameters.getOptions(); @@ -235,16 +244,21 @@ public class DatasyncHook { } List excludes = parameters.getExcludes(); if (excludes != null && excludes.size() > 0) { - List collect = excludes.stream().map(e->FilterRule.builder().filterType(e.getFilterType()).value(e.getValue()).build()).collect(Collectors.toList()); + List collect = excludes.stream() + .map(e -> FilterRule.builder().filterType(e.getFilterType()).value(e.getValue()).build()) + .collect(Collectors.toList()); builder.excludes(collect); } List includes = parameters.getIncludes(); if (includes != null && includes.size() > 0) { - List collect = includes.stream().map(e->FilterRule.builder().filterType(e.getFilterType()).value(e.getValue()).build()).collect(Collectors.toList()); + List collect = includes.stream() + .map(e -> FilterRule.builder().filterType(e.getFilterType()).value(e.getValue()).build()) + .collect(Collectors.toList()); builder.excludes(collect); } if (parameters.getSchedule() != null) { - builder.schedule(TaskSchedule.builder().scheduleExpression(parameters.getSchedule().getScheduleExpression()).build()); + builder.schedule(TaskSchedule.builder().scheduleExpression(parameters.getSchedule().getScheduleExpression()) + .build()); } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java index caa9bec5a5..d3197429b9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsHook.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.task.dms; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.commons.io.IOUtils; @@ -30,6 +30,10 @@ import java.util.Arrays; import java.util.Date; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,13 +60,11 @@ import com.amazonaws.services.databasemigrationservice.model.StopReplicationTask import com.amazonaws.services.databasemigrationservice.model.Tag; import com.amazonaws.services.databasemigrationservice.model.TestConnectionRequest; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - @Data public class DmsHook { - protected final Logger logger = LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); + + protected final Logger logger = + LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); private AWSDatabaseMigrationService client; private String replicationTaskIdentifier; private String sourceEndpointArn; @@ -93,27 +95,27 @@ public class DmsHook { // create a DMS client return AWSDatabaseMigrationServiceClientBuilder.standard() - .withCredentials(awsCredentialsProvider) - .withRegion(awsRegion) - .build(); + .withCredentials(awsCredentialsProvider) + .withRegion(awsRegion) + .build(); } public Boolean createReplicationTask() throws Exception { logger.info("createReplicationTask ......"); CreateReplicationTaskRequest request = new CreateReplicationTaskRequest() - .withReplicationTaskIdentifier(replicationTaskIdentifier) - .withSourceEndpointArn(sourceEndpointArn) - .withTargetEndpointArn(targetEndpointArn) - .withReplicationInstanceArn(replicationInstanceArn) - .withMigrationType(migrationType) - .withTableMappings(tableMappings) - .withReplicationTaskSettings(replicationTaskSettings) - .withCdcStartTime(cdcStartTime) - .withCdcStartPosition(cdcStartPosition) - .withCdcStopPosition(cdcStopPosition) - .withTags(tags) - .withTaskData(taskData) - .withResourceIdentifier(resourceIdentifier); + .withReplicationTaskIdentifier(replicationTaskIdentifier) + .withSourceEndpointArn(sourceEndpointArn) + .withTargetEndpointArn(targetEndpointArn) + .withReplicationInstanceArn(replicationInstanceArn) + .withMigrationType(migrationType) + .withTableMappings(tableMappings) + .withReplicationTaskSettings(replicationTaskSettings) + .withCdcStartTime(cdcStartTime) + .withCdcStartPosition(cdcStartPosition) + .withCdcStopPosition(cdcStopPosition) + .withTags(tags) + .withTaskData(taskData) + .withResourceIdentifier(resourceIdentifier); request.setTableMappings(replaceFileParameters(request.getTableMappings())); request.setReplicationTaskSettings(replaceFileParameters(request.getReplicationTaskSettings())); @@ -121,19 +123,19 @@ public class DmsHook { CreateReplicationTaskResult result = client.createReplicationTask(request); replicationTaskIdentifier = result.getReplicationTask().getReplicationTaskIdentifier(); replicationTaskArn = result.getReplicationTask().getReplicationTaskArn(); - logger.info("replicationTaskIdentifier: {}, replicationTaskArn: {}", replicationTaskIdentifier, replicationTaskArn); + logger.info("replicationTaskIdentifier: {}, replicationTaskArn: {}", replicationTaskIdentifier, + replicationTaskArn); return awaitReplicationTaskStatus(STATUS.READY); } - public Boolean startReplicationTask() { logger.info("startReplicationTask ......"); StartReplicationTaskRequest request = new StartReplicationTaskRequest() - .withReplicationTaskArn(replicationTaskArn) - .withStartReplicationTaskType(startReplicationTaskType) - .withCdcStartTime(cdcStartTime) - .withCdcStartPosition(cdcStartPosition) - .withCdcStopPosition(cdcStopPosition); + .withReplicationTaskArn(replicationTaskArn) + .withStartReplicationTaskType(startReplicationTaskType) + .withCdcStartTime(cdcStartTime) + .withCdcStartPosition(cdcStartPosition) + .withCdcStopPosition(cdcStopPosition); StartReplicationTaskResult result = client.startReplicationTask(request); replicationTaskArn = result.getReplicationTask().getReplicationTaskArn(); return awaitReplicationTaskStatus(STATUS.RUNNING); @@ -152,7 +154,7 @@ public class DmsHook { return; } StopReplicationTaskRequest request = new StopReplicationTaskRequest() - .withReplicationTaskArn(replicationTaskArn); + .withReplicationTaskArn(replicationTaskArn); client.stopReplicationTask(request); awaitReplicationTaskStatus(STATUS.STOPPED); } @@ -160,7 +162,7 @@ public class DmsHook { public Boolean deleteReplicationTask() { logger.info("deleteReplicationTask ......"); DeleteReplicationTaskRequest request = new DeleteReplicationTaskRequest() - .withReplicationTaskArn(replicationTaskArn); + .withReplicationTaskArn(replicationTaskArn); client.deleteReplicationTask(request); Boolean isDeleteSuccessfully; try { @@ -172,14 +174,14 @@ public class DmsHook { } public Boolean testConnectionEndpoint() { - return (testConnection(replicationInstanceArn, sourceEndpointArn) && testConnection(replicationInstanceArn, targetEndpointArn)); + return (testConnection(replicationInstanceArn, sourceEndpointArn) + && testConnection(replicationInstanceArn, targetEndpointArn)); } public Boolean testConnection(String replicationInstanceArn, String endpointArn) { logger.info("Test connect replication instance: {} and endpoint: {}", replicationInstanceArn, endpointArn); - TestConnectionRequest request = new TestConnectionRequest(). - withReplicationInstanceArn(replicationInstanceArn) - .withEndpointArn(endpointArn); + TestConnectionRequest request = new TestConnectionRequest().withReplicationInstanceArn(replicationInstanceArn) + .withEndpointArn(endpointArn); try { client.testConnection(request); } catch (InvalidResourceStateException e) { @@ -190,10 +192,12 @@ public class DmsHook { } public Boolean awaitConnectSuccess(String replicationInstanceArn, String endpointArn) { - Filter instanceFilters = new Filter().withName(AWS_KEY.REPLICATION_INSTANCE_ARN).withValues(replicationInstanceArn); + Filter instanceFilters = + new Filter().withName(AWS_KEY.REPLICATION_INSTANCE_ARN).withValues(replicationInstanceArn); Filter endpointFilters = new Filter().withName(AWS_KEY.ENDPOINT_ARN).withValues(endpointArn); - DescribeConnectionsRequest request = new DescribeConnectionsRequest().withFilters(endpointFilters, instanceFilters) - .withMarker(""); + DescribeConnectionsRequest request = + new DescribeConnectionsRequest().withFilters(endpointFilters, instanceFilters) + .withMarker(""); while (true) { ThreadUtils.sleep(CONSTANTS.CHECK_INTERVAL); DescribeConnectionsResult response = client.describeConnections(request); @@ -210,8 +214,10 @@ public class DmsHook { } public ReplicationTask describeReplicationTasks() { - Filter replicationTaskFilter = new Filter().withName(AWS_KEY.REPLICATION_TASK_ARN).withValues(replicationTaskArn); - DescribeReplicationTasksRequest request = new DescribeReplicationTasksRequest().withFilters(replicationTaskFilter).withMaxRecords(20).withMarker(""); + Filter replicationTaskFilter = + new Filter().withName(AWS_KEY.REPLICATION_TASK_ARN).withValues(replicationTaskArn); + DescribeReplicationTasksRequest request = new DescribeReplicationTasksRequest() + .withFilters(replicationTaskFilter).withMaxRecords(20).withMarker(""); DescribeReplicationTasksResult result = client.describeReplicationTasks(request); ReplicationTask replicationTask = result.getReplicationTasks().get(0); @@ -291,10 +297,12 @@ public class DmsHook { @AllArgsConstructor @NoArgsConstructor public static class ApplicationIds { + private String replicationTaskArn; } public static class STATUS { + public static final String DELETE = "delete"; public static final String READY = "ready"; public static final String RUNNING = "running"; @@ -305,17 +313,20 @@ public class DmsHook { } public static class AWS_KEY { + public static final String REPLICATION_TASK_ARN = "replication-task-arn"; public static final String REPLICATION_INSTANCE_ARN = "replication-instance-arn"; public static final String ENDPOINT_ARN = "endpoint-arn"; } public static class START_TYPE { + public static final String START_REPLICATION = "start-replication"; public static final String RELOAD_TARGET = "reload-target"; } public static class CONSTANTS { + public static final int CHECK_INTERVAL = 1000; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java index a68637216f..3adce9e1b5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java @@ -21,14 +21,12 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; - +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; -import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import java.util.TimeZone; @@ -52,7 +50,6 @@ public abstract class AbstractEmrTask extends AbstractRemoteTask { AmazonElasticMapReduce emrClient; String clusterId; - /** * config ObjectMapper features and propertyNamingStrategy * use UpperCamelCaseStrategy support capital letters parse @@ -60,12 +57,12 @@ public abstract class AbstractEmrTask extends AbstractRemoteTask { * @see PropertyNamingStrategy.UpperCamelCaseStrategy */ static final ObjectMapper objectMapper = new ObjectMapper() - .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) - .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) - .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) - .configure(REQUIRE_SETTERS_FOR_GETTERS, true) - .setTimeZone(TimeZone.getDefault()) - .setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); + .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) + .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .setTimeZone(TimeZone.getDefault()) + .setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); /** * constructor @@ -107,8 +104,8 @@ public abstract class AbstractEmrTask extends AbstractRemoteTask { final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials); // create an EMR client return AmazonElasticMapReduceClientBuilder.standard() - .withCredentials(awsCredentialsProvider) - .withRegion(awsRegion) - .build(); + .withCredentials(awsCredentialsProvider) + .withRegion(awsRegion) + .build(); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java index a1b9ec430f..6da4b2029e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.jupyter; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; @@ -30,7 +31,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.spi.utils.DateUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.io.IOException; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java index a460841d4c..72048f0e34 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTask.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.mlflow; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; @@ -31,7 +32,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.OSUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.ArrayList; @@ -167,18 +167,19 @@ public class MlflowTask extends AbstractTask { Map paramsMap = getParamsMap(); List args = new ArrayList<>(); - args.add(String.format(MlflowConstants.EXPORT_MLFLOW_TRACKING_URI_ENV, mlflowParameters.getMlflowTrackingUri())); + args.add( + String.format(MlflowConstants.EXPORT_MLFLOW_TRACKING_URI_ENV, mlflowParameters.getMlflowTrackingUri())); String runCommand; String versionString; if (mlflowParameters.isCustomProject()) { - versionString = getVersionString(mlflowParameters.getMlflowProjectVersion(), mlflowParameters.getMlflowProjectRepository()); + versionString = getVersionString(mlflowParameters.getMlflowProjectVersion(), + mlflowParameters.getMlflowProjectRepository()); } else { versionString = getVersionString(getPresetRepositoryVersion(), getPresetRepository()); } - switch (mlflowParameters.getMlflowJobType()) { case MlflowConstants.JOB_TYPE_BASIC_ALGORITHM: args.add(String.format(MlflowConstants.SET_DATA_PATH, mlflowParameters.getDataPath())); @@ -187,8 +188,9 @@ public class MlflowTask extends AbstractTask { args.add(String.format(MlflowConstants.SET_REPOSITORY, repoBasicAlgorithm)); runCommand = MlflowConstants.MLFLOW_RUN_BASIC_ALGORITHM; - runCommand = String.format(runCommand, mlflowParameters.getAlgorithm(), mlflowParameters.getParams(), mlflowParameters.getSearchParams(), mlflowParameters.getModelName(), - mlflowParameters.getExperimentName()); + runCommand = String.format(runCommand, mlflowParameters.getAlgorithm(), mlflowParameters.getParams(), + mlflowParameters.getSearchParams(), mlflowParameters.getModelName(), + mlflowParameters.getExperimentName()); break; case MlflowConstants.JOB_TYPE_AUTOML: @@ -197,13 +199,15 @@ public class MlflowTask extends AbstractTask { args.add(String.format(MlflowConstants.SET_REPOSITORY, repoAutoML)); runCommand = MlflowConstants.MLFLOW_RUN_AUTOML_PROJECT; - runCommand = String.format(runCommand, mlflowParameters.getAutomlTool(), mlflowParameters.getParams(), mlflowParameters.getModelName(), mlflowParameters.getExperimentName()); + runCommand = String.format(runCommand, mlflowParameters.getAutomlTool(), mlflowParameters.getParams(), + mlflowParameters.getModelName(), mlflowParameters.getExperimentName()); break; case MlflowConstants.JOB_TYPE_CUSTOM_PROJECT: args.add(String.format(MlflowConstants.SET_REPOSITORY, mlflowParameters.getMlflowProjectRepository())); runCommand = MlflowConstants.MLFLOW_RUN_CUSTOM_PROJECT; - runCommand = String.format(runCommand, mlflowParameters.getParams(), mlflowParameters.getExperimentName()); + runCommand = + String.format(runCommand, mlflowParameters.getParams(), mlflowParameters.getExperimentName()); break; default: @@ -226,11 +230,13 @@ public class MlflowTask extends AbstractTask { Map paramsMap = getParamsMap(); List args = new ArrayList<>(); - args.add(String.format(MlflowConstants.EXPORT_MLFLOW_TRACKING_URI_ENV, mlflowParameters.getMlflowTrackingUri())); + args.add( + String.format(MlflowConstants.EXPORT_MLFLOW_TRACKING_URI_ENV, mlflowParameters.getMlflowTrackingUri())); String deployModelKey = mlflowParameters.getDeployModelKey(); if (mlflowParameters.getDeployType().equals(MlflowConstants.MLFLOW_MODELS_DEPLOY_TYPE_MLFLOW)) { - args.add(String.format(MlflowConstants.MLFLOW_MODELS_SERVE, deployModelKey, mlflowParameters.getDeployPort())); + args.add(String.format(MlflowConstants.MLFLOW_MODELS_SERVE, deployModelKey, + mlflowParameters.getDeployPort())); } else if (mlflowParameters.getDeployType().equals(MlflowConstants.MLFLOW_MODELS_DEPLOY_TYPE_DOCKER)) { String imageName = "mlflow/" + mlflowParameters.getModelKeyName(":"); @@ -238,7 +244,8 @@ public class MlflowTask extends AbstractTask { args.add(String.format(MlflowConstants.MLFLOW_BUILD_DOCKER, deployModelKey, imageName)); args.add(String.format(MlflowConstants.DOCKER_RREMOVE_CONTAINER, containerName)); - args.add(String.format(MlflowConstants.DOCKER_RUN, containerName, mlflowParameters.getDeployPort(), imageName)); + args.add(String.format(MlflowConstants.DOCKER_RUN, containerName, mlflowParameters.getDeployPort(), + imageName)); } return ParameterUtils.convertParameterPlaceholders(String.join("\n", args), ParamUtils.convert(paramsMap)); @@ -253,7 +260,8 @@ public class MlflowTask extends AbstractTask { public int checkDockerHealth() { logger.info("checking container healthy ... "); int exitCode = -1; - String[] command = {"sh", "-c", String.format(MlflowConstants.DOCKER_HEALTH_CHECK, mlflowParameters.getContainerName())}; + String[] command = + {"sh", "-c", String.format(MlflowConstants.DOCKER_HEALTH_CHECK, mlflowParameters.getContainerName())}; for (int x = 0; x < MlflowConstants.DOCKER_HEALTH_CHECK_TIMEOUT; x = x + 1) { String status; try { @@ -268,7 +276,8 @@ public class MlflowTask extends AbstractTask { logger.info("container is healthy"); return exitCode; } else { - logger.info("The health check has been running for {} seconds", x * MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL / 1000); + logger.info("The health check has been running for {} seconds", + x * MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL / 1000); ThreadUtils.sleep(MlflowConstants.DOCKER_HEALTH_CHECK_INTERVAL); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java index c49b189d1a..2a6f0f23b1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/test/java/org/apache/dolphinler/plugin/task/mlflow/MlflowTaskTest.java @@ -17,13 +17,13 @@ package org.apache.dolphinler.plugin.task.mlflow; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowConstants; import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowParameters; import org.apache.dolphinscheduler.plugin.task.mlflow.MlflowTask; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java index 8299f85cd2..bd3a48267f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTask.java @@ -22,6 +22,7 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; @@ -30,7 +31,6 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.dolphinscheduler.spi.utils.PropertyUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.Collections; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 555b5bc770..8f8a27372b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.api.ProcessUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; @@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.task.TaskPluginManager; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java index a00c8d67a6..58e1fe7e88 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java @@ -21,9 +21,9 @@ import lombok.NonNull; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.BaseCommand; import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java index d738732334..d85181f1ed 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java @@ -17,10 +17,8 @@ package org.apache.dolphinscheduler.server.worker.processor; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -35,9 +33,10 @@ import org.apache.dolphinscheduler.server.worker.runner.WorkerDelayTaskExecuteRu import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnableFactoryBuilder; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.utils.LogUtils; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java index bff92e5d11..4f6eb9ce47 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java @@ -20,12 +20,12 @@ package org.apache.dolphinscheduler.server.worker.processor; import com.google.common.base.Preconditions; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java index a78d8c8f2a..fc8ead96fa 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java @@ -18,13 +18,12 @@ package org.apache.dolphinscheduler.server.worker.processor; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckMessage; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; - +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index 3ebf497cda..52af436221 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.worker.processor; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; @@ -36,6 +35,7 @@ import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; import org.apache.dolphinscheduler.service.log.LogClient; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils; import java.util.Arrays; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java index b6eb954b5d..7fefb61729 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java @@ -20,12 +20,12 @@ package org.apache.dolphinscheduler.server.worker.processor; import com.google.common.base.Preconditions; import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskRejectAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java index 1a621a39e7..b264eb0110 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java @@ -22,7 +22,6 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; @@ -34,6 +33,7 @@ import org.apache.dolphinscheduler.remote.command.TaskSavePointResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java index adc989cc5a..c00fb4f99a 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java @@ -18,13 +18,13 @@ package org.apache.dolphinscheduler.server.worker.runner; import lombok.NonNull; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java index 53de57ec68..49262b3900 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java @@ -18,11 +18,11 @@ package org.apache.dolphinscheduler.server.worker.runner; import lombok.NonNull; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java index 6e6e854ee4..19351ee764 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.server.worker.runner; import lombok.NonNull; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java index 44bb8878d4..7db96fc380 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java @@ -18,11 +18,11 @@ package org.apache.dolphinscheduler.server.worker.runner; import lombok.NonNull; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java index d858ee38c5..ebc358b026 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java @@ -21,11 +21,8 @@ import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.storage.StorageOperate; -import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; @@ -42,7 +39,10 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.server.worker.utils.TaskExecutionCheckerUtils; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; +import org.apache.dolphinscheduler.service.utils.CommonUtils; +import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils; import javax.annotation.Nullable; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java index f3edfd4c1d..fee1635be0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java @@ -19,11 +19,11 @@ package org.apache.dolphinscheduler.server.worker.runner; import lombok.NonNull; import lombok.experimental.UtilityClass; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java index 56d9481642..d0467a25dc 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java @@ -18,8 +18,6 @@ package org.apache.dolphinscheduler.server.worker.utils; import org.apache.dolphinscheduler.common.exception.StorageOperateNoConfiguredException; -import org.apache.dolphinscheduler.common.storage.StorageOperate; -import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -27,7 +25,8 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; - +import org.apache.dolphinscheduler.service.storage.StorageOperate; +import org.apache.dolphinscheduler.service.utils.CommonUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.SystemUtils; @@ -53,7 +52,7 @@ public class TaskExecutionCheckerUtils { if (workerConfig.isTenantDistributedUser() && SystemUtils.IS_OS_LINUX) { // use the id command to judge in linux osUserExistFlag = OSUtils.existTenantCodeInLinux(taskExecutionContext.getTenantCode()); - } else if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) { + } else if (OSUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) { // if not exists this user, then create OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode()); osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode()); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java index 66ae632288..55b1a5b5dd 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import java.util.Date; diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java index ca9af127f7..3ad1f27b74 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.junit.Test; import org.junit.jupiter.api.Assertions; @@ -95,4 +95,4 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { Assertions.assertAll(workerTaskExecuteRunnable::run); Assertions.assertEquals(TaskExecutionStatus.FAILURE, taskExecutionContext.getCurrentExecutionStatus()); } -} \ No newline at end of file +}