From d46e4659d917e1994bc9045c57fe3b865a308352 Mon Sep 17 00:00:00 2001 From: Eric Gao Date: Wed, 19 Oct 2022 11:08:04 +0800 Subject: [PATCH] [Improvement][Style] Fix remaining style errors and wildcard imports (#12412) * Fix remaining style errors and wildcard imports * Upgrade spotless to the latest stable version, exclude .github folder from spotless check and fix dead link * Fix spotless matching scope of markdown files --- docs/docs/en/about/glossary.md | 2 +- docs/docs/en/about/hardware.md | 3 +- .../en/contribute/join/become-a-committer.md | 3 +- docs/docs/en/guide/datasource/oracle.md | 24 +- docs/docs/en/guide/installation/standalone.md | 4 +- docs/docs/en/guide/upgrade/incompatible.md | 1 + docs/docs/zh/about/hardware.md | 3 +- .../zh/contribute/join/become-a-committer.md | 1 + docs/docs/zh/guide/datasource/oracle.md | 1 + docs/docs/zh/guide/parameter/built-in.md | 61 ++- docs/docs/zh/guide/upgrade/incompatible.md | 1 + .../dolphinscheduler-alert-api/pom.xml | 11 +- .../alert/api/AlertChannel.java | 1 + .../alert/api/AlertChannelFactory.java | 3 +- .../alert/api/AlertConstants.java | 1 + .../dolphinscheduler/alert/api/ShowType.java | 1 + .../dolphinscheduler-alert-dingtalk/pom.xml | 7 +- .../dingtalk/DingTalkAlertChannelFactory.java | 19 +- .../dingtalk/DingTalkParamsConstants.java | 1 + .../plugin/alert/dingtalk/DingTalkSender.java | 17 +- .../dolphinscheduler-alert-email/pom.xml | 7 +- .../plugin/alert/email/EmailAlertChannel.java | 1 + .../alert/email/EmailAlertChannelFactory.java | 142 ++--- .../plugin/alert/email/EmailConstants.java | 28 +- .../plugin/alert/email/ExcelUtils.java | 18 +- .../alert/email/MailParamsConstants.java | 1 + .../plugin/alert/email/MailSender.java | 20 +- .../email/exception/AlertEmailException.java | 1 + .../email/template/DefaultHTMLTemplate.java | 10 +- .../dolphinscheduler-alert-feishu/pom.xml | 7 +- .../alert/feishu/FeiShuAlertChannel.java | 1 + .../feishu/FeiShuAlertChannelFactory.java | 62 ++- .../alert/feishu/FeiShuParamsConstants.java | 1 + .../plugin/alert/feishu/FeiShuSender.java | 14 +- .../plugin/alert/feishu/HttpRequestUtil.java | 4 +- .../dolphinscheduler-alert-http/pom.xml | 7 +- .../plugin/alert/http/HttpAlertChannel.java | 1 + .../alert/http/HttpAlertChannelFactory.java | 63 ++- .../plugin/alert/http/HttpAlertConstants.java | 1 + .../dolphinscheduler-alert-pagerduty/pom.xml | 7 +- .../pagerduty/PagerDutyAlertChannel.java | 1 + .../PagerDutyAlertChannelFactory.java | 9 +- .../pagerduty/PagerDutyParamsConstants.java | 1 + .../alert/pagerduty/PagerDutySender.java | 20 +- .../dolphinscheduler-alert-script/pom.xml | 7 +- .../plugin/alert/script/OSUtils.java | 1 + .../plugin/alert/script/ProcessUtils.java | 1 + .../alert/script/ScriptAlertChannel.java | 1 + .../script/ScriptAlertChannelFactory.java | 41 +- .../alert/script/ScriptParamsConstants.java | 1 + .../plugin/alert/script/ScriptType.java | 1 + .../plugin/alert/script/StreamGobbler.java | 1 + .../dolphinscheduler-alert-slack/pom.xml | 7 +- .../plugin/alert/slack/SlackAlertChannel.java | 1 + .../alert/slack/SlackAlertChannelFactory.java | 24 +- .../alert/slack/SlackParamsConstants.java | 1 + .../plugin/alert/slack/SlackSender.java | 16 +- .../dolphinscheduler-alert-telegram/pom.xml | 7 +- .../alert/telegram/TelegramAlertChannel.java | 1 + .../telegram/TelegramAlertChannelFactory.java | 74 ++- .../telegram/TelegramParamsConstants.java | 1 - .../plugin/alert/telegram/TelegramSender.java | 14 +- .../dolphinscheduler-alert-webexteams/pom.xml | 7 +- .../plugin/alert/webexteams/WebexMessage.java | 1 + .../webexteams/WebexTeamsAlertChannel.java | 1 + .../WebexTeamsAlertChannelFactory.java | 90 +-- .../webexteams/WebexTeamsDestination.java | 4 +- .../alert/webexteams/WebexTeamsSender.java | 9 +- .../dolphinscheduler-alert-wechat/pom.xml | 7 +- .../alert/wechat/WeChatAlertChannel.java | 1 + .../wechat/WeChatAlertChannelFactory.java | 74 +-- .../alert/wechat/WeChatAlertConstants.java | 4 +- .../wechat/WeChatAlertParamsConstants.java | 1 + .../plugin/alert/wechat/WeChatSender.java | 28 +- .../plugin/alert/wechat/WeChatType.java | 2 +- .../alert/wechat/WechatAppChatMessage.java | 4 +- .../plugin/alert/wechat/WechatAppMessage.java | 3 +- .../dolphinscheduler-alert-plugins/pom.xml | 7 +- .../alert/AlertPluginManager.java | 32 +- .../alert/AlertRequestProcessor.java | 14 +- .../alert/AlertServerMetrics.java | 2 +- dolphinscheduler-alert/pom.xml | 10 +- .../api/ApiApplicationServer.java | 1 - .../api/aspect/AccessLogAnnotation.java | 1 + .../api/aspect/AccessLogAspect.java | 16 +- .../api/audit/AuditMessage.java | 4 +- .../api/configuration/AuditConfiguration.java | 3 +- .../OpenAPITranslationConfiguration.java | 44 +- .../PythonGatewayConfiguration.java | 1 + .../configuration/TaskTypeConfiguration.java | 12 +- .../configuration/TrafficConfiguration.java | 4 +- .../ApiFuncIdentificationConstant.java | 23 +- .../api/controller/AccessTokenController.java | 2 +- .../api/controller/AlertGroupController.java | 2 +- .../AlertPluginInstanceController.java | 1 + .../api/controller/ClusterController.java | 2 +- .../api/controller/DataQualityController.java | 2 +- .../api/controller/DataSourceController.java | 2 +- .../api/controller/DsErrorController.java | 2 + .../api/controller/EnvironmentController.java | 2 +- .../api/controller/ExecutorController.java | 26 +- .../api/controller/FavTaskController.java | 22 +- .../controller/K8sNamespaceController.java | 2 +- .../api/controller/MonitorController.java | 3 +- .../ProcessDefinitionController.java | 12 +- .../controller/ProcessInstanceController.java | 42 +- .../api/controller/ProjectController.java | 2 +- .../api/controller/ProjectV2Controller.java | 2 +- .../api/controller/QueueController.java | 2 +- .../api/controller/QueueV2Controller.java | 24 +- .../api/controller/ResourcesController.java | 2 +- .../api/controller/SchedulerController.java | 1 + .../controller/TaskDefinitionController.java | 2 +- .../controller/TaskInstanceController.java | 2 +- .../api/controller/TenantController.java | 2 +- .../api/controller/UsersController.java | 2 +- .../controller/WorkFlowLineageController.java | 2 +- .../api/controller/WorkerGroupController.java | 20 +- .../api/dto/CommandStateCount.java | 3 +- .../api/dto/CreateTokenRequest.java | 2 +- .../api/dto/CreateTokenResponse.java | 1 + .../api/dto/RuleDefinition.java | 2 +- .../api/dto/TaskCountDto.java | 8 +- .../api/dto/TaskStateCount.java | 3 +- .../api/dto/gantt/GanttDto.java | 9 +- .../dolphinscheduler/api/dto/gantt/Task.java | 1 + .../api/dto/project/ProjectCreateRequest.java | 2 +- .../dto/project/ProjectDeleteResponse.java | 2 +- .../api/dto/project/ProjectQueryRequest.java | 3 +- .../api/dto/project/ProjectUpdateRequest.java | 4 +- .../api/dto/queue/QueueCreateRequest.java | 2 +- .../api/dto/queue/QueueQueryRequest.java | 3 +- .../api/dto/queue/QueueUpdateRequest.java | 3 +- .../api/dto/queue/QueueVerifyRequest.java | 2 +- .../api/dto/resources/FileLeaf.java | 2 +- .../api/dto/resources/filter/IFilter.java | 1 + .../dto/resources/filter/ResourceFilter.java | 17 +- .../visitor/ResourceTreeVisitor.java | 27 +- .../api/dto/resources/visitor/Visitor.java | 1 + .../dto/schedule/ScheduleCreateRequest.java | 3 - .../api/enums/ExecuteType.java | 2 +- .../api/exceptions/ApiException.java | 1 + .../api/exceptions/ApiExceptionHandler.java | 1 - .../api/interceptor/RateLimitInterceptor.java | 16 +- .../api/k8s/K8sClientService.java | 35 +- .../dolphinscheduler/api/k8s/K8sManager.java | 4 +- .../api/permission/PermissionCheck.java | 10 +- .../ResourcePermissionCheckService.java | 11 +- .../ResourcePermissionCheckServiceImpl.java | 46 +- .../api/python/PythonGateway.java | 144 +++-- .../api/security/Authenticator.java | 1 + .../api/security/SecurityConfig.java | 1 + .../security/impl/AbstractAuthenticator.java | 3 +- .../security/impl/ldap/LdapAuthenticator.java | 2 +- .../api/service/AccessTokenService.java | 1 - .../api/service/AlertGroupService.java | 3 +- .../service/AlertPluginInstanceService.java | 5 +- .../api/service/BaseService.java | 9 +- .../api/service/ClusterService.java | 1 - .../api/service/DataAnalysisService.java | 4 +- .../api/service/DataSourceService.java | 2 +- .../api/service/DqRuleService.java | 2 +- .../api/service/EnvironmentService.java | 5 +- ...EnvironmentWorkerGroupRelationService.java | 1 - .../api/service/ExecutorService.java | 3 +- .../api/service/K8sNamespaceService.java | 8 +- .../api/service/LoggerService.java | 1 - .../api/service/MonitorService.java | 12 +- .../api/service/ProcessDefinitionService.java | 4 +- .../api/service/ProcessInstanceService.java | 10 +- .../api/service/QueueService.java | 2 - .../api/service/ResourcesService.java | 17 +- .../api/service/TaskGroupQueueService.java | 8 +- .../api/service/TaskGroupService.java | 2 +- .../api/service/TenantService.java | 2 +- .../api/service/UdfFuncService.java | 22 +- .../api/service/UsersService.java | 10 +- .../api/service/WorkerGroupService.java | 3 +- .../service/impl/AccessTokenServiceImpl.java | 7 +- .../service/impl/AlertGroupServiceImpl.java | 44 +- .../impl/AlertPluginInstanceServiceImpl.java | 46 +- .../api/service/impl/AuditServiceImpl.java | 2 +- .../api/service/impl/BaseServiceImpl.java | 55 +- .../api/service/impl/ClusterServiceImpl.java | 4 +- .../service/impl/DataAnalysisServiceImpl.java | 16 +- .../service/impl/DataSourceServiceImpl.java | 77 +-- .../impl/DqExecuteResultServiceImpl.java | 1 - .../service/impl/EnvironmentServiceImpl.java | 45 +- ...ronmentWorkerGroupRelationServiceImpl.java | 12 +- .../api/service/impl/FavTaskServiceImpl.java | 6 +- .../service/impl/K8SNamespaceServiceImpl.java | 4 +- .../api/service/impl/LoggerServiceImpl.java | 17 +- .../api/service/impl/MonitorServiceImpl.java | 50 +- .../impl/ProcessDefinitionServiceImpl.java | 6 +- .../impl/ProcessInstanceServiceImpl.java | 144 +++-- .../api/service/impl/ProjectServiceImpl.java | 34 +- .../api/service/impl/QueueServiceImpl.java | 29 +- .../service/impl/ResourcesServiceImpl.java | 70 ++- .../api/service/impl/SessionServiceImpl.java | 2 +- .../impl/TaskGroupQueueServiceImpl.java | 21 +- .../service/impl/TaskGroupServiceImpl.java | 3 +- .../service/impl/TaskInstanceServiceImpl.java | 45 +- .../api/service/impl/TenantServiceImpl.java | 64 ++- .../api/service/impl/UdfFuncServiceImpl.java | 76 +-- .../api/service/impl/UsersServiceImpl.java | 22 +- .../impl/WorkFlowLineageServiceImpl.java | 60 +- .../service/impl/WorkerGroupServiceImpl.java | 3 +- .../api/utils/CheckUtils.java | 8 +- .../dolphinscheduler/api/utils/FileUtils.java | 1 + .../dolphinscheduler/api/utils/PageInfo.java | 4 +- .../controller/WorkerGroupControllerTest.java | 4 +- .../api/service/BaseServiceTest.java | 1 + .../service/ProcessInstanceServiceTest.java | 23 +- .../ProcessTaskRelationServiceTest.java | 1 + .../api/service/TenantServiceTest.java | 3 +- .../api/service/WorkerGroupServiceTest.java | 8 +- .../config/YamlPropertySourceFactory.java | 6 +- .../common/enums/AlertEvent.java | 2 +- .../common/enums/AlertStatus.java | 1 + .../common/enums/AlertType.java | 4 +- .../common/enums/AlertWarnLevel.java | 2 +- .../common/enums/AuditResourceType.java | 1 + .../common/enums/AuthorizationType.java | 17 +- .../common/enums/BlockingOpportunity.java | 2 +- .../common/enums/CacheType.java | 1 + .../common/enums/CommandType.java | 2 +- .../common/enums/ComplementDependentMode.java | 5 +- .../common/enums/ConditionType.java | 1 + .../dolphinscheduler/common/enums/Flag.java | 1 + .../common/enums/HttpCheckCondition.java | 4 +- .../common/enums/HttpMethod.java | 2 +- .../common/enums/HttpParametersType.java | 2 +- .../common/enums/PluginType.java | 3 +- .../common/enums/Priority.java | 1 + .../common/enums/ReleaseState.java | 4 +- .../common/enums/RunMode.java | 3 +- .../common/enums/SlotCheckState.java | 2 +- .../common/enums/SqoopJobType.java | 3 +- .../common/enums/TaskDependType.java | 1 + .../common/enums/TaskExecuteType.java | 1 + .../common/enums/TaskRecordStatus.java | 3 +- .../common/enums/TimeoutFlag.java | 1 + .../common/enums/UdfType.java | 14 +- .../common/enums/UserType.java | 2 +- .../common/enums/WarningType.java | 3 +- .../common/enums/WorkflowExecutionStatus.java | 8 +- .../StorageOperateNoConfiguredException.java | 3 +- .../dolphinscheduler/common/graph/DAG.java | 19 +- .../common/model/BaseHeartBeatTask.java | 6 +- .../common/model/MasterHeartBeat.java | 1 + .../common/model/PreviousTaskNode.java | 2 +- .../dolphinscheduler/common/model/Server.java | 4 +- .../common/model/TaskNodeRelation.java | 2 +- .../common/model/WorkerHeartBeat.java | 1 - .../common/process/HttpProperty.java | 9 +- .../common/shell/AbstractShell.java | 28 +- .../common/shell/ShellExecutor.java | 12 +- .../common/thread/ThreadUtils.java | 4 +- .../common/utils/CollectionUtils.java | 3 +- .../common/utils/DateUtils.java | 19 +- .../common/utils/FileUtils.java | 12 +- .../common/utils/HttpUtils.java | 18 +- .../common/utils/KerberosHttpClient.java | 15 +- .../common/utils/RetryerUtils.java | 4 +- .../utils/placeholder/BusinessTimeUtils.java | 4 +- .../common/ConstantsTest.java | 1 + .../common/exception/ExceptionTest.java | 3 +- .../common/graph/DAGTest.java | 511 +++++++++--------- .../common/os/OSUtilsTest.java | 8 +- .../common/utils/CodeGenerateUtilsTest.java | 5 +- .../common/utils/DateUtilsTest.java | 17 +- .../common/utils/EncryptionUtilsTest.java | 1 - .../common/utils/FileUtilsTest.java | 11 +- .../common/utils/LocalJettyHttpServer.java | 21 +- .../utils/LocalServerHttpUtilsTest.java | 16 +- .../common/utils/NetUtilsTest.java | 3 +- .../common/utils/PropertyUtilsTest.java | 1 + .../common/utils/RetryerUtilsTest.java | 7 +- .../common/utils/ScriptRunnerTest.java | 7 +- .../common/utils/StringTest.java | 12 +- .../apache/dolphinscheduler/dao/AlertDao.java | 4 +- .../dao/entity/CycleDependency.java | 1 + .../dolphinscheduler/dao/entity/DagData.java | 3 +- .../dao/entity/DataSource.java | 2 +- .../entity/DependentProcessDefinition.java | 6 +- .../entity/DqExecuteResultAlertContent.java | 5 +- .../dao/entity/ErrorCommand.java | 2 +- .../dao/entity/ExecuteStatusCount.java | 3 +- .../dolphinscheduler/dao/entity/FavTask.java | 7 +- .../dao/entity/ProcessAlertContent.java | 20 +- .../dao/entity/ResponseTaskLog.java | 1 + .../dao/entity/ServerAlertContent.java | 1 + .../dolphinscheduler/dao/entity/Session.java | 6 +- .../dao/entity/TaskAlertContent.java | 9 +- .../dao/entity/WorkFlowLineage.java | 1 + .../dao/entity/WorkFlowRelation.java | 3 +- .../dao/entity/ZookeeperRecord.java | 6 +- .../dao/mapper/AccessTokenMapper.java | 7 +- .../dao/mapper/AlertGroupMapper.java | 10 +- .../dao/mapper/AuditLogMapper.java | 1 + .../dao/mapper/CommandMapper.java | 11 +- .../dao/mapper/DataSourceMapper.java | 15 +- .../dao/mapper/DataSourceUserMapper.java | 1 - .../dao/mapper/EnvironmentMapper.java | 6 +- .../dao/mapper/ErrorCommandMapper.java | 6 +- .../dao/mapper/FavTaskMapper.java | 4 +- .../dao/mapper/K8sNamespaceMapper.java | 167 +++--- .../mapper/ProcessDefinitionLogMapper.java | 7 +- .../dao/mapper/ProcessInstanceMapMapper.java | 4 +- .../dao/mapper/ProjectMapper.java | 4 +- .../dao/mapper/QueueMapper.java | 10 +- .../dao/mapper/ResourceMapper.java | 12 +- .../dao/mapper/ResourceUserMapper.java | 2 +- .../dao/mapper/SessionMapper.java | 6 +- .../dao/mapper/TaskDefinitionLogMapper.java | 3 +- .../dao/mapper/TaskGroupMapper.java | 9 +- .../dao/mapper/TaskGroupQueueMapper.java | 19 +- .../dao/mapper/TaskInstanceMapper.java | 31 +- .../dao/mapper/TenantMapper.java | 12 +- .../dao/mapper/UDFUserMapper.java | 5 +- .../dao/mapper/UdfFuncMapper.java | 10 +- .../dao/mapper/UserMapper.java | 1 - .../dao/mapper/WorkFlowLineageMapper.java | 7 +- .../dao/upgrade/JsonSplitDao.java | 45 +- .../dao/upgrade/ProcessDefinitionDao.java | 7 +- .../dao/upgrade/ScheduleDao.java | 3 +- .../dao/upgrade/SchemaUtils.java | 7 +- .../dao/upgrade/WorkerGroupDao.java | 16 +- .../dao/utils/BeanContext.java | 4 +- .../dao/utils/PostgreSQLPerformance.java | 4 +- .../utils/ResourceProcessDefinitionUtils.java | 4 +- .../dao/utils/TaskInstanceUtils.java | 3 +- .../dao/utils/WorkflowUtils.java | 12 +- .../dolphinscheduler/dao/AlertDaoTest.java | 11 +- .../dao/entity/ProcessDefinitionTest.java | 8 +- .../dao/entity/TaskInstanceTest.java | 8 +- .../dao/mapper/AccessTokenMapperTest.java | 9 +- .../dao/mapper/AlertGroupMapperTest.java | 13 +- .../dao/mapper/AlertSendStatusMapperTest.java | 1 + .../dao/mapper/AuditLogMapperTest.java | 2 +- .../dao/mapper/CommandMapperTest.java | 1 - .../dao/mapper/DataSourceUserMapperTest.java | 10 +- .../mapper/DqRuleInputEntryMapperTest.java | 9 +- ...ironmentWorkerGroupRelationMapperTest.java | 10 +- .../dao/mapper/ErrorCommandMapperTest.java | 12 +- .../dao/mapper/PluginDefineTest.java | 3 +- .../mapper/ProcessInstanceMapMapperTest.java | 17 +- .../dao/mapper/ProcessInstanceMapperTest.java | 15 +- .../ProcessTaskRelationLogMapperTest.java | 2 +- .../mapper/ProcessTaskRelationMapperTest.java | 2 +- .../dao/mapper/ProjectMapperTest.java | 27 +- .../dao/mapper/ProjectUserMapperTest.java | 13 +- .../dao/mapper/QueueMapperTest.java | 14 +- .../dao/mapper/ResourceUserMapperTest.java | 23 +- .../dao/mapper/SessionMapperTest.java | 14 +- .../dao/mapper/TaskDefinitionMapperTest.java | 2 +- .../dao/mapper/TaskGroupMapperTest.java | 1 - .../dao/mapper/TaskInstanceMapperTest.java | 4 +- .../dao/mapper/TenantMapperTest.java | 22 +- .../dao/mapper/UDFUserMapperTest.java | 45 +- .../dao/mapper/UserMapperTest.java | 48 +- .../dao/mapper/WorkFlowLineageMapperTest.java | 20 +- .../dao/mapper/WorkerGroupMapperTest.java | 11 +- .../dao/upgrade/ProcessDefinitionDaoTest.java | 18 +- .../dao/upgrade/SchemaUtilsTest.java | 2 +- .../dao/upgrade/WorkerGroupDaoTest.java | 9 +- .../ResourceProcessDefinitionUtilsTest.java | 19 +- .../dao/utils/TaskInstanceUtilsTest.java | 2 +- .../dao/utils/WorkflowUtilsTest.java | 6 +- .../data/quality/DataQualityApplication.java | 10 +- .../data/quality/config/BaseConfig.java | 4 +- .../data/quality/config/Config.java | 8 +- .../config/DataQualityConfiguration.java | 5 +- .../data/quality/config/EnvConfig.java | 4 +- .../data/quality/config/ReaderConfig.java | 5 +- .../quality/config/TransformerConfig.java | 5 +- .../data/quality/config/WriterConfig.java | 5 +- .../quality/context/DataQualityContext.java | 8 +- .../data/quality/enums/ReaderType.java | 3 +- .../data/quality/enums/TransformerType.java | 3 +- .../data/quality/enums/WriterType.java | 3 +- .../data/quality/execution/Execution.java | 1 + .../execution/SparkBatchExecution.java | 12 +- .../execution/SparkRuntimeEnvironment.java | 4 +- .../data/quality/flow/Component.java | 4 +- .../quality/flow/batch/reader/HiveReader.java | 2 +- .../quality/flow/batch/reader/JdbcReader.java | 4 +- .../flow/batch/reader/ReaderFactory.java | 4 +- .../batch/transformer/TransformerFactory.java | 6 +- .../quality/flow/batch/writer/JdbcWriter.java | 18 +- .../flow/batch/writer/WriterFactory.java | 6 +- .../batch/writer/file/BaseFileWriter.java | 13 +- .../batch/writer/file/HdfsFileWriter.java | 2 +- .../batch/writer/file/LocalFileWriter.java | 2 +- .../data/quality/utils/JsonUtils.java | 4 +- .../dolphinscheduler-datasource-all/pom.xml | 7 +- .../api/client/CommonDataSourceClient.java | 5 +- .../datasource/BaseHDFSConnectionParam.java | 1 + .../api/datasource/DataSourceProcessor.java | 1 - .../api/plugin/DataSourcePluginManager.java | 11 +- .../plugin/DataSourceProcessorManager.java | 1 + .../plugin/DataSourceProcessorProvider.java | 6 +- .../datasource/api/utils/CommonUtils.java | 10 +- .../datasource/api/utils/DataSourceUtils.java | 11 +- .../api/datasource/MySQLConnectionParam.java | 1 + .../pom.xml | 10 +- .../athena/AthenaDataSourceChannel.java | 3 +- .../AthenaDataSourceChannelFactory.java | 1 + .../athena/AthenaDataSourceClient.java | 1 + .../param/AthenaDataSourceParamDTO.java | 1 + .../param/AthenaDataSourceProcessor.java | 20 +- .../ClickHouseDataSourceChannelFactory.java | 1 + .../param/ClickHouseConnectionParam.java | 1 + .../param/ClickHouseDataSourceProcessor.java | 9 +- .../db2/param/Db2ConnectionParam.java | 1 + .../db2/param/Db2DataSourceProcessor.java | 3 +- .../hive/HiveDataSourceChannelFactory.java | 1 + .../hive/param/HiveConnectionParam.java | 1 + .../hive/param/HiveDataSourceProcessor.java | 3 +- .../mysql/MySQLDataSourceChannelFactory.java | 1 + .../mysql/param/MySQLConnectionParam.java | 1 + .../mysql/param/MySQLDataSourceProcessor.java | 18 +- .../OracleDataSourceChannelFactory.java | 1 + .../oracle/param/OracleConnectionParam.java | 1 + .../PostgreSQLDataSourceChannelFactory.java | 1 + .../param/PostgreSQLConnectionParam.java | 1 + .../param/PostgreSQLDataSourceProcessor.java | 9 +- .../PrestoDataSourceChannelFactory.java | 1 + .../presto/param/PrestoConnectionParam.java | 1 + .../param/PrestoDataSourceProcessor.java | 3 +- .../pom.xml | 9 +- .../redshift/RedshiftDataSourceChannel.java | 3 +- .../RedshiftDataSourceChannelFactory.java | 1 + .../redshift/RedshiftDataSourceClient.java | 1 + .../param/RedshiftConnectionParam.java | 1 + .../param/RedshiftDataSourceProcessor.java | 15 +- .../spark/SparkDataSourceChannelFactory.java | 1 + .../spark/param/SparkConnectionParam.java | 1 + .../spark/param/SparkDataSourceProcessor.java | 15 +- .../SQLServerDataSourceChannelFactory.java | 1 + .../param/SQLServerConnectionParam.java | 1 + .../param/SQLServerDataSourceProcessor.java | 3 +- .../server/master/MasterServer.java | 5 +- .../builder/TaskExecutionContextBuilder.java | 5 +- .../ProcessInstanceExecCacheManager.java | 3 +- .../StreamTaskInstanceExecCacheManager.java | 3 +- .../ProcessInstanceExecCacheManagerImpl.java | 6 +- ...treamTaskInstanceExecCacheManagerImpl.java | 11 +- .../server/master/config/MasterConfig.java | 12 +- .../consumer/TaskPriorityQueueConsumer.java | 15 +- .../controller/WorkflowExecuteController.java | 3 +- .../master/dispatch/ExecutorDispatcher.java | 2 +- .../executor/NettyExecutorManager.java | 2 - .../dispatch/host/CommonHostManager.java | 5 +- .../dispatch/host/LowerWeightHostManager.java | 13 +- .../host/assign/AbstractSelector.java | 5 +- .../dispatch/host/assign/HostWeight.java | 15 +- .../host/assign/LowerWeightRoundRobin.java | 12 +- .../host/assign/RoundRobinSelector.java | 4 +- .../server/master/event/StateEvent.java | 6 +- .../master/event/StateEventHandler.java | 4 +- .../event/StateEventHandlerManager.java | 4 +- .../event/TaskRetryStateEventHandler.java | 6 +- .../master/event/TaskRunningEventHandler.java | 5 +- .../server/master/event/TaskStateEvent.java | 7 +- .../event/TaskTimeoutStateEventHandler.java | 26 +- .../event/TaskWaitTaskGroupStateHandler.java | 3 +- .../event/WorkflowBlockStateEventHandler.java | 8 +- .../event/WorkflowStartEventHandler.java | 28 +- .../master/event/WorkflowStateEvent.java | 9 +- .../event/WorkflowStateEventHandler.java | 4 +- .../WorkflowTimeoutStateEventHandler.java | 3 +- .../master/metrics/MasterServerMetrics.java | 2 +- .../metrics/ProcessInstanceMetrics.java | 26 +- .../server/master/metrics/TaskMetrics.java | 14 +- .../master/processor/CacheProcessor.java | 4 +- .../HostUpdateResponseProcessor.java | 7 +- .../master/processor/StateEventProcessor.java | 6 +- .../master/processor/TaskEventProcessor.java | 6 +- .../TaskExecuteResponseProcessor.java | 12 +- .../TaskExecuteRunningProcessor.java | 11 +- .../processor/TaskExecuteStartProcessor.java | 15 +- .../processor/TaskKillResponseProcessor.java | 9 +- .../master/processor/TaskRecallProcessor.java | 8 +- ...WorkflowExecutingDataRequestProcessor.java | 10 +- .../queue/StateEventResponseService.java | 17 +- .../processor/queue/TaskEventService.java | 16 +- .../processor/queue/TaskExecuteRunnable.java | 10 +- .../queue/TaskExecuteThreadPool.java | 21 +- .../MasterConnectionStateListener.java | 4 +- .../master/registry/MasterRegistryClient.java | 10 +- .../registry/MasterRegistryDataListener.java | 4 +- .../master/registry/MasterStopStrategy.java | 1 + .../registry/MasterWaitingStrategy.java | 5 +- .../master/runner/EventExecuteService.java | 1 + .../master/runner/FailoverExecuteThread.java | 1 + .../runner/MasterSchedulerBootstrap.java | 4 +- .../runner/StateWheelExecuteThread.java | 15 +- .../runner/StreamTaskExecuteThreadPool.java | 4 +- .../master/runner/WorkflowEventLooper.java | 12 +- .../runner/WorkflowExecuteRunnable.java | 20 +- .../runner/WorkflowExecuteThreadPool.java | 14 +- .../runner/task/CommonTaskProcessor.java | 16 +- .../master/runner/task/TaskInstanceKey.java | 7 +- .../runner/task/TaskProcessorFactory.java | 19 +- .../master/service/ExecutingService.java | 3 +- .../master/service/FailoverService.java | 5 +- .../master/task/MasterHeartBeatTask.java | 5 +- .../server/master/utils/DependentExecute.java | 6 +- .../server/master/utils/SwitchTaskUtils.java | 3 +- .../server/master/ConditionsTaskTest.java | 11 +- .../server/master/DependentTaskTest.java | 9 +- .../server/master/ParamsTest.java | 10 +- .../server/master/SwitchTaskTest.java | 13 +- ...ocessInstanceExecCacheManagerImplTest.java | 2 +- .../TaskPriorityQueueConsumerTest.java | 2 +- .../dispatch/ExecutorDispatcherTest.java | 2 +- .../executor/NettyExecutorManagerTest.java | 20 +- .../host/RoundRobinHostManagerTest.java | 3 +- .../assign/LowerWeightRoundRobinTest.java | 30 +- .../host/assign/RandomSelectorTest.java | 6 +- .../host/assign/RoundRobinSelectorTest.java | 13 +- .../processor/TaskAckProcessorTest.java | 2 +- .../TaskResponseProcessorTestConfig.java | 2 +- .../queue/TaskResponseServiceTest.java | 2 +- .../runner/task/CommonTaskProcessorTest.java | 36 +- dolphinscheduler-meter/pom.xml | 33 +- .../meter/MeterConfiguration.java | 1 + dolphinscheduler-microbench/pom.xml | 32 +- .../base/AbstractBaseBenchmark.java | 1 - .../microbench/common/EnumBenchMark.java | 23 +- .../microbench/common/RpcTest.java | 1 + .../microbench/common/UserCallback.java | 1 + dolphinscheduler-python/pom.xml | 17 +- .../pydolphinscheduler/DEVELOP.md | 4 +- .../dolphinscheduler-registry-all/pom.xml | 7 +- .../dolphinscheduler-registry-api/pom.xml | 8 +- .../api/ConnectStrategyProperties.java | 4 +- .../registry/api/ConnectionListener.java | 1 + .../dolphinscheduler/registry/api/Event.java | 8 +- .../registry/api/SubscribeListener.java | 1 + .../dolphinscheduler-registry-etcd/pom.xml | 9 +- .../etcd/EtcdConnectionStateListener.java | 2 +- .../plugin/registry/etcd/EtcdRegistry.java | 44 +- .../registry/etcd/EtcdRegistryProperties.java | 5 +- .../dolphinscheduler-registry-mysql/pom.xml | 7 +- .../plugin/registry/mysql/MysqlRegistry.java | 14 +- .../mysql/MysqlRegistryProperties.java | 4 +- .../plugin/registry/mysql/model/DataType.java | 1 + .../mysql/task/RegistryLockManager.java | 8 +- .../mysql/task/SubscribeDataManager.java | 11 +- .../ZookeeperConnectionStateListener.java | 2 +- .../registry/zookeeper/ZookeeperRegistry.java | 27 +- .../ZookeeperRegistryProperties.java | 3 + .../dolphinscheduler-registry-plugins/pom.xml | 6 +- dolphinscheduler-registry/pom.xml | 41 +- .../remote/NettyRemotingClient.java | 61 ++- .../remote/NettyRemotingServer.java | 15 +- .../remote/codec/NettyDecoder.java | 1 + .../remote/codec/NettyEncoder.java | 1 - .../remote/command/Command.java | 4 +- .../remote/command/CommandType.java | 1 - .../remote/command/HostUpdateCommand.java | 3 +- .../command/StateEventResponseCommand.java | 5 +- .../remote/command/TaskDispatchCommand.java | 1 - .../command/TaskEventChangeCommand.java | 5 +- .../command/TaskExecuteResultCommand.java | 4 - .../command/TaskExecuteRunningAckMessage.java | 7 +- .../command/TaskExecuteRunningCommand.java | 2 +- .../command/TaskKillResponseCommand.java | 9 +- .../command/TaskSavePointResponseCommand.java | 8 +- .../WorkflowStateEventChangeCommand.java | 5 +- .../alert/AlertSendRequestCommand.java | 7 +- .../alert/AlertSendResponseCommand.java | 7 +- .../alert/AlertSendResponseResult.java | 4 +- .../command/log/GetAppIdRequestCommand.java | 7 +- .../command/log/GetAppIdResponseCommand.java | 7 +- .../log/GetLogBytesRequestCommand.java | 7 +- .../log/GetLogBytesResponseCommand.java | 7 +- .../log/RemoveTaskLogRequestCommand.java | 7 +- .../log/RemoveTaskLogResponseCommand.java | 7 +- .../log/RollViewLogRequestCommand.java | 7 +- .../log/RollViewLogResponseCommand.java | 7 +- .../command/log/ViewLogRequestCommand.java | 7 +- .../command/log/ViewLogResponseCommand.java | 7 +- .../remote/config/NettyClientConfig.java | 3 +- .../remote/dto/TaskInstanceExecuteDto.java | 5 +- .../remote/dto/WorkflowExecuteDto.java | 8 +- .../remote/exceptions/RemotingException.java | 1 - .../exceptions/RemotingTimeoutException.java | 4 +- .../RemotingTooMuchRequestException.java | 2 +- .../remote/future/ReleaseSemaphore.java | 6 +- .../remote/future/ResponseFuture.java | 3 +- .../remote/handler/NettyClientHandler.java | 6 +- .../remote/handler/NettyServerHandler.java | 7 +- .../remote/processor/NettyRemoteChannel.java | 5 +- .../processor/NettyRequestProcessor.java | 3 +- .../processor/StateEventCallbackService.java | 3 +- .../utils/CallerThreadExecutePolicy.java | 6 +- .../remote/utils/Constants.java | 1 - .../remote/utils/JsonSerializer.java | 1 + .../remote/utils/NamedThreadFactory.java | 4 +- .../apache/dolphinscheduler/rpc/base/Rpc.java | 5 +- .../dolphinscheduler/rpc/base/RpcService.java | 1 + .../rpc/client/ConsumerInterceptor.java | 8 +- .../rpc/codec/NettyDecoder.java | 3 +- .../rpc/codec/NettyEncoder.java | 3 +- .../rpc/common/RequestEventType.java | 5 +- .../rpc/common/ResponseEventType.java | 30 +- .../rpc/common/ThreadPoolManager.java | 3 +- .../rpc/protocol/EventType.java | 6 +- .../rpc/remote/NettyClient.java | 8 +- .../rpc/remote/NettyClientHandler.java | 4 +- .../rpc/remote/NettyServer.java | 23 +- .../rpc/serializer/ProtoStuffSerializer.java | 30 +- .../rpc/serializer/RpcSerializer.java | 31 +- .../remote/NettyRemotingClientTest.java | 33 +- .../alert/AlertSendRequestCommandTest.java | 4 +- .../alert/AlertSendResponseCommandTest.java | 5 +- .../command/cache/CacheExpireCommandTest.java | 1 + .../command/future/ResponseFutureTest.java | 10 +- .../log/GetLogBytesRequestCommandTest.java | 1 + .../log/GetLogBytesResponseCommandTest.java | 1 + .../log/RemoveTaskLogRequestCommandTest.java | 1 + .../log/RemoveTaskLogResponseCommandTest.java | 1 + .../log/RollViewLogRequestCommandTest.java | 1 + .../apache/dolphinscheduler/rpc/RpcTest.java | 1 + .../dolphinscheduler-scheduler-api/pom.xml | 7 +- .../scheduler/api/SchedulerApi.java | 2 +- .../dolphinscheduler-scheduler-quartz/pom.xml | 7 +- .../scheduler/quartz/ProcessScheduleTask.java | 14 +- .../scheduler/quartz/QuartzScheduler.java | 19 +- dolphinscheduler-scheduler-plugin/pom.xml | 11 +- .../bean/SpringApplicationContext.java | 2 +- .../service/cache/CacheNotifyService.java | 1 + .../service/cron/AbstractCycle.java | 2 +- .../service/cron/CronUtils.java | 38 +- .../service/cron/CycleFactory.java | 348 ++++++------ .../service/cron/CycleLinks.java | 86 +-- .../service/expand/CuringParamsService.java | 12 +- .../service/json/DateJsonSerializer.java | 1 + .../service/queue/MasterPriorityQueue.java | 1 + .../queue/PeerTaskInstancePriorityQueue.java | 11 +- .../service/queue/TaskPriority.java | 73 +-- .../service/queue/TaskPriorityQueue.java | 3 +- .../service/queue/TaskPriorityQueueImpl.java | 2 +- .../service/task/TaskPluginManager.java | 8 +- .../service/utils/ProcessData.java | 6 +- .../service/alert/AlertClientServiceTest.java | 9 +- .../alert/ProcessAlertManagerTest.java | 9 +- .../service/cache/CacheNotifyServiceTest.java | 7 +- .../service/cron/CronUtilsTest.java | 80 +-- .../expand/CuringGlobalParamsServiceTest.java | 52 +- ...ePlaceholderResolverExpandServiceTest.java | 4 +- .../service/log/LogClientTest.java | 5 +- .../log/LoggerRequestProcessorTest.java | 4 +- .../service/log/MasterLogFilterTest.java | 8 +- .../log/SensitiveDataConverterTest.java | 9 +- .../service/log/TaskLogDiscriminatorTest.java | 6 +- .../service/log/TaskLogFilterTest.java | 8 +- .../service/log/WorkerLogFilterTest.java | 8 +- .../service/process/ProcessServiceTest.java | 70 ++- .../PeerTaskInstancePriorityQueueTest.java | 8 +- .../queue/TaskPriorityQueueImplTest.java | 49 +- .../service/storage/impl/HadoopUtilsTest.java | 1 + .../service/storage/impl/OssOperatorTest.java | 22 +- .../service/utils/CommonUtilsTest.java | 7 +- .../service/utils/LogUtilsTest.java | 12 +- .../service/utils/ProcessUtilsTest.java | 11 +- .../datasource/DataSourceChannelFactory.java | 1 + .../spi/enums/CommandType.java | 2 +- .../dolphinscheduler/spi/enums/DbType.java | 8 +- .../dolphinscheduler/spi/enums/Flag.java | 1 + .../spi/enums/ResourceType.java | 1 + .../spi/params/PluginParamsTransfer.java | 10 +- .../spi/params/base/ParamsOptions.java | 3 +- .../spi/params/base/PluginParams.java | 15 +- .../spi/params/base/Validate.java | 1 + .../spi/params/fswitch/SwitchParam.java | 2 +- .../spi/params/group/GroupParam.java | 6 +- .../spi/params/input/InputParam.java | 2 +- .../spi/params/radio/RadioParam.java | 4 +- .../spi/plugin/PrioritySPIFactory.java | 9 +- .../dolphinscheduler/spi/utils/Constants.java | 8 +- .../dolphinscheduler/spi/utils/DateUtils.java | 4 +- .../dolphinscheduler/spi/utils/JSONUtils.java | 21 +- .../spi/utils/StringUtils.java | 27 +- .../spi/params/PluginParamsTransferTest.java | 67 +-- .../spi/plugin/PrioritySPIFactoryTest.java | 12 +- .../spi/utils/JSONUtilsTest.java | 26 +- .../dolphinscheduler/StandaloneServer.java | 4 +- .../dolphinscheduler-task-all/pom.xml | 2 +- .../plugin/task/api/AbstractShell.java | 19 +- .../task/api/K8sTaskExecutionContext.java | 5 +- .../plugin/task/api/ProcessUtils.java | 5 +- .../task/api/SQLTaskExecutionContext.java | 1 - .../plugin/task/api/ShellCommandExecutor.java | 6 +- .../plugin/task/api/ShellExecutor.java | 4 +- .../plugin/task/api/TaskChannelFactory.java | 2 +- .../plugin/task/api/TaskConstants.java | 2 - .../plugin/task/api/enums/DataType.java | 2 +- .../plugin/task/api/enums/DependResult.java | 1 - .../task/api/enums/DependentRelation.java | 2 +- .../plugin/task/api/enums/Direct.java | 2 +- .../task/api/enums/TaskTimeoutStrategy.java | 1 + .../plugin/task/api/enums/UdfType.java | 1 + .../plugin/task/api/enums/dp/CheckType.java | 13 +- .../task/api/enums/dp/ConnectorType.java | 9 +- .../task/api/enums/dp/DqFailureStrategy.java | 3 +- .../plugin/task/api/enums/dp/DqTaskState.java | 11 +- .../task/api/enums/dp/ExecuteSqlType.java | 11 +- .../plugin/task/api/enums/dp/InputType.java | 13 +- .../task/api/enums/dp/OperatorType.java | 17 +- .../task/api/enums/dp/OptionSourceType.java | 13 +- .../plugin/task/api/enums/dp/RuleType.java | 13 +- .../plugin/task/api/enums/dp/ValueType.java | 13 +- .../plugin/task/api/k8s/AbstractK8sTask.java | 6 +- .../task/api/k8s/AbstractK8sTaskExecutor.java | 1 + .../task/api/k8s/K8sTaskMainParameters.java | 14 +- .../task/api/loop/LoopTaskDefinition.java | 4 +- .../task/api/loop/LoopTaskInstanceStatus.java | 1 + .../api/loop/LoopTaskMethodDefinition.java | 3 +- .../loop/template/LoopTaskYamlDefinition.java | 5 + .../template/TemplateMethodTransformer.java | 3 +- .../template/http/HttpLoopTaskDefinition.java | 5 +- .../http/HttpLoopTaskMethodDefinition.java | 1 + ...ttpLoopTaskCancelTaskMethodDefinition.java | 5 +- ...tpLoopTaskQueryStatusMethodDefinition.java | 5 +- ...ttpLoopTaskSubmitTaskMethodDefinition.java | 11 +- .../CancelTemplateMethodTransformer.java | 15 +- .../http/parser/HttpTaskDefinitionParser.java | 16 +- .../QueryStateTemplateMethodTransformer.java | 17 +- .../SubmitTemplateMethodTransformer.java | 17 +- .../plugin/task/api/model/DateInterval.java | 1 + .../plugin/task/api/model/DependentItem.java | 3 +- .../task/api/model/DependentTaskModel.java | 3 +- .../plugin/task/api/model/Property.java | 2 +- .../plugin/task/api/model/SwitchResultVo.java | 2 +- .../api/parameters/AbstractParameters.java | 3 +- .../api/parameters/ConditionsParameters.java | 8 +- .../api/parameters/DependentParameters.java | 5 +- .../task/api/parameters/IParameters.java | 31 +- .../api/parameters/K8sTaskParameters.java | 15 +- .../task/api/parameters/ParametersNode.java | 1 + .../api/parameters/SubProcessParameters.java | 2 +- .../task/api/parameters/SwitchParameters.java | 2 +- .../dataquality/DataQualityParameters.java | 4 +- .../resource/AbstractResourceParameters.java | 1 - .../resource/DataSourceParameters.java | 3 +- .../task/api/parser/BusinessTimeUtils.java | 1 + .../plugin/task/api/parser/ParamUtils.java | 8 +- .../task/api/parser/ParameterUtils.java | 13 +- .../task/api/parser/PlaceholderUtils.java | 10 +- .../api/parser/PropertyPlaceholderHelper.java | 27 +- .../task/api/parser/TimePlaceholderUtils.java | 69 +-- .../plugin/task/api/stream/StreamTask.java | 1 + .../task/api/stream/StreamTaskChannel.java | 3 - .../task/api/utils/DependentDateUtils.java | 2 +- .../plugin/task/api/utils/JsonPathUtils.java | 8 +- .../plugin/task/api/utils/OkHttpUtils.java | 6 +- .../plugin/task/api/utils/RetryUtils.java | 1 + .../plugin/task/api/model/JSONUtilsTest.java | 6 +- .../task/api/utils/ParameterUtilsTest.java | 3 +- .../dolphinscheduler-task-blocking/pom.xml | 7 +- .../blocking/BlockingTaskChannelFactory.java | 1 + .../dolphinscheduler-task-chunjun/pom.xml | 7 +- .../task/chunjun/ChunJunParameters.java | 4 +- .../plugin/task/chunjun/ChunJunTask.java | 29 +- .../dolphinscheduler-task-conditions/pom.xml | 9 +- .../ConditionsTaskChannelFactory.java | 1 + .../dolphinscheduler-task-dataquality/pom.xml | 7 +- .../task/dq/DataQualityTaskChannel.java | 2 +- .../dq/exception/DataQualityException.java | 3 +- .../plugin/task/dq/rule/RuleManager.java | 14 +- .../task/dq/rule/parameter/BaseConfig.java | 4 +- .../parameter/DataQualityConfiguration.java | 3 +- .../task/dq/rule/parameter/EnvConfig.java | 4 +- .../task/dq/rule/parser/IRuleParser.java | 2 +- .../task/dq/rule/parser/MappingColumn.java | 3 +- .../parser/MultiTableAccuracyRuleParser.java | 23 +- .../MultiTableComparisonRuleParser.java | 7 +- .../SingleTableCustomSqlRuleParser.java | 22 +- .../dq/rule/parser/SingleTableRuleParser.java | 22 +- .../plugin/task/dq/utils/RuleParserUtils.java | 311 ++++++----- .../plugin/task/dq/utils/SparkArgsUtils.java | 5 +- .../dolphinscheduler-task-datasync/pom.xml | 9 +- .../task/datasync/DatasyncParameters.java | 27 +- .../task/datasync/DatasyncTaskChannel.java | 2 +- .../datasync/DatasyncTaskChannelFactory.java | 5 +- .../dolphinscheduler-task-datax/pom.xml | 7 +- .../plugin/task/datax/DataxParameters.java | 7 +- .../plugin/task/datax/DataxTask.java | 58 +- .../plugin/task/datax/DataxUtils.java | 2 +- .../plugin/task/datax/DataxTaskTest.java | 48 +- .../dolphinscheduler-task-dependent/pom.xml | 9 +- .../task/dependent/DependentTaskChannel.java | 3 +- .../DependentTaskChannelFactory.java | 1 + .../dolphinscheduler-task-dinky/pom.xml | 11 +- .../plugin/task/dinky/DinkyParameters.java | 8 +- .../task/dinky/DinkyTaskChannelFactory.java | 5 +- .../plugin/task/dinky/DinkyTaskConstants.java | 5 +- .../dolphinscheduler-task-dms/pom.xml | 7 +- .../plugin/task/dms/DmsParameters.java | 6 +- .../plugin/task/dms/DmsTask.java | 19 +- .../task/dms/DmsTaskChannelFactory.java | 1 - .../dolphinscheduler-task-dvc/pom.xml | 7 +- .../plugin/task/dvc/DvcConstants.java | 18 +- .../plugin/task/dvc/DvcParameters.java | 21 +- .../plugin/task/dvc/DvcTaskChannel.java | 2 - .../task/dvc/DvcTaskChannelFactory.java | 9 +- .../dolphinscheduler-task-emr/pom.xml | 7 +- .../plugin/task/emr/AbstractEmrTask.java | 1 + .../plugin/task/emr/EmrParameters.java | 15 +- .../plugin/task/emr/EmrTaskChannel.java | 1 + .../task/emr/EmrTaskChannelFactory.java | 1 + .../pom.xml | 9 +- .../task/flink/FlinkStreamTaskChannel.java | 2 +- .../flink/FlinkStreamTaskChannelFactory.java | 1 + .../dolphinscheduler-task-flink/pom.xml | 27 +- .../plugin/task/flink/FileUtils.java | 28 +- .../plugin/task/flink/FlinkArgsUtils.java | 65 ++- .../plugin/task/flink/FlinkTaskChannel.java | 1 + .../task/flink/FlinkTaskChannelFactory.java | 1 + .../dolphinscheduler-task-http/pom.xml | 8 +- .../plugin/task/http/HttpCheckCondition.java | 2 +- .../plugin/task/http/HttpParameters.java | 1 + .../plugin/task/http/HttpParametersType.java | 2 +- .../plugin/task/http/HttpProperty.java | 1 + .../plugin/task/http/HttpTask.java | 17 +- .../dolphinscheduler-task-java/pom.xml | 11 +- .../plugin/task/java/JavaConstants.java | 2 - .../plugin/task/java/JavaParameters.java | 1 + .../plugin/task/java/JavaTask.java | 14 +- .../task/java/JavaTaskChannelFactory.java | 1 + .../JavaSourceFileExistException.java | 4 +- .../PublicClassNotFoundException.java | 4 +- .../dolphinscheduler-task-jupyter/pom.xml | 7 +- .../jupyter/JupyterTaskChannelFactory.java | 1 + .../dolphinscheduler-task-k8s/pom.xml | 7 +- .../plugin/task/k8s/K8sTaskChannel.java | 1 + .../task/k8s/K8sTaskChannelFactory.java | 1 + .../dolphinscheduler-task-mlflow/pom.xml | 7 +- .../plugin/task/mlflow/MlflowConstants.java | 31 +- .../plugin/task/mlflow/MlflowParameters.java | 11 +- .../plugin/task/mlflow/MlflowTaskChannel.java | 1 - .../task/mlflow/MlflowTaskChannelFactory.java | 8 +- .../dolphinscheduler-task-mr/pom.xml | 7 +- .../plugin/task/mr/MapReduceParameters.java | 3 +- .../plugin/task/mr/MapReduceTask.java | 14 +- .../plugin/task/mr/MapReduceTaskChannel.java | 1 + .../dolphinscheduler-task-openmldb/pom.xml | 7 +- .../task/openmldb/OpenmldbTaskChannel.java | 1 + .../openmldb/OpenmldbTaskChannelFactory.java | 1 + .../plugin/task/pigeon/PigeonConfig.java | 3 +- .../plugin/task/pigeon/PigeonTaskChannel.java | 1 + .../task/pigeon/PigeonTaskChannelFactory.java | 8 +- .../dolphinscheduler-task-procedure/pom.xml | 7 +- .../task/procedure/ProcedureParameters.java | 6 +- .../dolphinscheduler-task-python/pom.xml | 7 +- .../plugin/task/python/PythonParameters.java | 1 + .../plugin/task/python/PythonTaskChannel.java | 1 + .../task/python/PythonTaskChannelFactory.java | 31 +- .../dolphinscheduler-task-pytorch/pom.xml | 7 +- .../task/pytorch/GitProjectManager.java | 8 +- .../task/pytorch/PytorchParameters.java | 2 - .../plugin/task/pytorch/PytorchTask.java | 10 +- .../pytorch/PytorchTaskChannelFactory.java | 1 + .../task/sagemaker/SagemakerConstants.java | 1 + .../SagemakerTaskChannelFactory.java | 1 + .../dolphinscheduler-task-seatunnel/pom.xml | 7 +- .../plugin/task/seatunnel/DeployModeEnum.java | 1 + .../task/seatunnel/SeatunnelParameters.java | 7 +- .../task/seatunnel/SeatunnelTaskChannel.java | 3 +- .../flink/SeatunnelFlinkParameters.java | 1 + .../seatunnel/flink/SeatunnelFlinkTask.java | 7 +- .../spark/SeatunnelSparkParameters.java | 1 + .../seatunnel/spark/SeatunnelSparkTask.java | 9 +- .../dolphinscheduler-task-shell/pom.xml | 7 +- .../plugin/task/shell/ShellTask.java | 9 +- .../task/shell/ShellTaskChannelFactory.java | 1 + .../dolphinscheduler-task-spark/pom.xml | 7 +- .../task/spark/SparkTaskChannelFactory.java | 1 + .../dolphinscheduler-task-sql/pom.xml | 7 +- .../plugin/task/sql/SqlBinds.java | 1 + .../plugin/task/sql/SqlTask.java | 60 +- .../plugin/task/sql/SqlTaskChannel.java | 1 + .../task/sql/SqlTaskChannelFactory.java | 1 + .../dolphinscheduler-task-sqoop/pom.xml | 7 +- .../plugin/task/sqoop/SqoopConstants.java | 14 +- .../plugin/task/sqoop/SqoopJobType.java | 2 +- .../plugin/task/sqoop/SqoopTask.java | 11 +- .../task/sqoop/generator/CommonGenerator.java | 28 +- .../sqoop/generator/SqoopJobGenerator.java | 7 +- .../sources/HdfsSourceGenerator.java | 6 +- .../sources/HiveSourceGenerator.java | 16 +- .../sources/MySQLSourceGenerator.java | 29 +- .../targets/HdfsTargetGenerator.java | 12 +- .../targets/HiveTargetGenerator.java | 22 +- .../targets/MySQLTargetGenerator.java | 24 +- .../task/sqoop/parameter/SqoopParameters.java | 34 +- .../plugin/task/sqoop/EntityTestUtils.java | 12 +- .../dolphinscheduler-task-subprocess/pom.xml | 9 +- .../SubProcessTaskChannelFactory.java | 1 + .../dolphinscheduler-task-switch/pom.xml | 9 +- .../task/switchtask/SwitchParameters.java | 7 +- .../plugin/task/switchtask/SwitchResult.java | 1 + .../switchtask/SwitchTaskChannelFactory.java | 4 +- .../task/zeppelin/ZeppelinParameters.java | 7 +- .../zeppelin/ZeppelinTaskChannelFactory.java | 1 + dolphinscheduler-tools/pom.xml | 23 +- .../datasource/DolphinSchedulerManager.java | 19 +- .../datasource/InitDolphinScheduler.java | 2 + .../datasource/UpgradeDolphinScheduler.java | 2 + .../tools/datasource/dao/MySQLUpgradeDao.java | 13 +- .../datasource/dao/PostgreSQLUpgradeDao.java | 2 +- .../tools/datasource/dao/ResourceDao.java | 10 +- dolphinscheduler-ui/pom.xml | 128 ++--- .../server/worker/WorkerServer.java | 11 +- .../worker/message/MessageRetryRunner.java | 17 +- .../TaskExecuteResultMessageSender.java | 8 +- .../message/TaskRejectMessageSender.java | 4 +- .../server/worker/metrics/TaskMetrics.java | 6 +- .../worker/metrics/WorkerServerMetrics.java | 28 +- .../worker/processor/HostUpdateProcessor.java | 3 +- .../processor/TaskDispatchProcessor.java | 16 +- .../TaskExecuteResultAckProcessor.java | 6 +- .../TaskExecuteRunningAckProcessor.java | 2 +- .../processor/TaskRejectAckProcessor.java | 6 +- .../processor/TaskSavePointProcessor.java | 31 +- .../WorkerConnectionStateListener.java | 4 +- .../worker/registry/WorkerStopStrategy.java | 1 + .../registry/WorkerWaitingStrategy.java | 5 +- .../worker/rpc/WorkerMessageSender.java | 6 +- .../server/worker/rpc/WorkerRpcClient.java | 2 +- ...DefaultWorkerDelayTaskExecuteRunnable.java | 6 +- ...WorkerDelayTaskExecuteRunnableFactory.java | 10 +- .../WorkerDelayTaskExecuteRunnable.java | 12 +- ...WorkerDelayTaskExecuteRunnableFactory.java | 22 +- .../worker/runner/WorkerExecService.java | 2 +- .../worker/runner/WorkerManagerThread.java | 13 +- .../runner/WorkerTaskExecuteRunnable.java | 13 +- ...rkerTaskExecuteRunnableFactoryBuilder.java | 5 +- .../utils/TaskExecutionCheckerUtils.java | 2 +- .../server/worker/config/BeanConfigTest.java | 3 +- .../processor/TaskDispatchProcessorTest.java | 4 +- ...ultWorkerDelayTaskExecuteRunnableTest.java | 12 +- pom.xml | 15 +- tools/release/README.md | 1 + 948 files changed, 5903 insertions(+), 4733 deletions(-) diff --git a/docs/docs/en/about/glossary.md b/docs/docs/en/about/glossary.md index a8f844c06f..29f0da5a5a 100644 --- a/docs/docs/en/about/glossary.md +++ b/docs/docs/en/about/glossary.md @@ -17,7 +17,7 @@ there are no subsequent nodes. Examples are as follows: manual start or scheduled scheduling. Each time the process definition runs, a process instance is generated **Task instance**: The task instance is the instantiation of the task node in the process definition, which identifies -the specific task +the specific task **Task type**: Currently supports SHELL, SQL, SUB_PROCESS (sub-process), PROCEDURE, MR, SPARK, PYTHON, DEPENDENT ( depends), and plans to support dynamic plug-in expansion, note: **SUB_PROCESS** need relation with another workflow definition which also a separate process diff --git a/docs/docs/en/about/hardware.md b/docs/docs/en/about/hardware.md index 97488332d4..604bb6a90e 100644 --- a/docs/docs/en/about/hardware.md +++ b/docs/docs/en/about/hardware.md @@ -49,4 +49,5 @@ DolphinScheduler provides the following network port configurations for normal o The minimum supported version of Google Chrome is version 85, but version 90 or above is recommended. ## Synchronize clocks -To avoid problems with internal cluster communications that can impact your task execution, make sure that the clocks on all of the cluster nodes are synchronized from a common clock source, such as using Chrony and/or NTP. Synchronizing the time ensures that every node in the cluster has the same time. \ No newline at end of file + +To avoid problems with internal cluster communications that can impact your task execution, make sure that the clocks on all of the cluster nodes are synchronized from a common clock source, such as using Chrony and/or NTP. Synchronizing the time ensures that every node in the cluster has the same time. diff --git a/docs/docs/en/contribute/join/become-a-committer.md b/docs/docs/en/contribute/join/become-a-committer.md index a64c854a0e..f748a1638c 100644 --- a/docs/docs/en/contribute/join/become-a-committer.md +++ b/docs/docs/en/contribute/join/become-a-committer.md @@ -20,7 +20,7 @@ In Dolphinscheduler, **new committer nomination** could only be officially start The following steps are recommended (to be initiated only by an existing PMC member): 1. Send an email titled `[DISCUSS] Promote xxx as new committer` to `private@dolphinscheduler.apache.org`. List the important contributions of the candidate, - so you could gather support from other PMC members for your proposal. +so you could gather support from other PMC members for your proposal. 2. Keep the discussion open for more than 3 days but no more than 1 week, unless there is any express objection or concern. 3. If the PMC generally agrees to the proposal, send an email titled `[VOTE] Promote xxx as new committer` to `private@dolphinscheduler.apache.org`. 4. Keep the voting process open for more than 3 days, but no more than 1 week. Consider the result as `Consensus Approval` if there are `3 + 1` votes with `NO` vetos. Please note that +1 votes > -1 votes. @@ -31,6 +31,7 @@ The following steps are recommended (to be initiated only by an existing PMC mem The PMC member who starts the promotion is responsible for sending an invitation to the new committer and guiding him/her to set up the ASF env. The PMC member should send an email using the following template to the new committer: + ``` To: @gmail.com Cc: private@dolphinscheduler.apache.org diff --git a/docs/docs/en/guide/datasource/oracle.md b/docs/docs/en/guide/datasource/oracle.md index f7ab96e1dd..122801a2d9 100644 --- a/docs/docs/en/guide/datasource/oracle.md +++ b/docs/docs/en/guide/datasource/oracle.md @@ -4,18 +4,18 @@ ## Datasource Parameters -| **Datasource** | **Description** | -|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| Datasource | Select Oracle. | -| Datasource Name | Enter the name of the datasource. | -| Description | Enter a description of the datasource. | -| IP/Host Name | Enter the Oracle service IP. | -| Port | Enter the Oracle service port. | -| Username | Set the username for Oracle connection. | -| Password | Set the password for Oracle connection. | -| Database Name | Enter the ServiceName or SID of the Oracle connection. | -| ServiceName or SID | Choose ServiceName or SID according to your entry in Database Name column. | -| jdbc connect parameters | Parameter settings for Oracle connection, in JSON format. For example, you can use {"schema": "abc"} to specify database abc for using. | +| **Datasource** | **Description** | +|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| Datasource | Select Oracle. | +| Datasource Name | Enter the name of the datasource. | +| Description | Enter a description of the datasource. | +| IP/Host Name | Enter the Oracle service IP. | +| Port | Enter the Oracle service port. | +| Username | Set the username for Oracle connection. | +| Password | Set the password for Oracle connection. | +| Database Name | Enter the ServiceName or SID of the Oracle connection. | +| ServiceName or SID | Choose ServiceName or SID according to your entry in Database Name column. | +| jdbc connect parameters | Parameter settings for Oracle connection, in JSON format. For example, you can use {"schema": "abc"} to specify database abc for using. | ## Native Supported diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index 30ee2a138f..d11f70f41e 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -2,8 +2,8 @@ Standalone only for quick experience for DolphinScheduler. -If you are a newbie and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). -If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). +If you are a newbie and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). +If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md). > **_Note:_** Standalone only recommends the usage of fewer than 20 workflows, because it uses in-memory H2 Database in default, ZooKeeper Testing Server, too many tasks may cause instability. diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 0fd2c409ea..ba1af107ce 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -11,3 +11,4 @@ This document records the incompatible updates between each version. You need to * Copy and import workflow without 'copy' suffix [#10607](https://github.com/apache/dolphinscheduler/pull/10607) * Use semicolon as default sql segment separator [#10869](https://github.com/apache/dolphinscheduler/pull/10869) + diff --git a/docs/docs/zh/about/hardware.md b/docs/docs/zh/about/hardware.md index 2ad98f4a6a..bba384925e 100644 --- a/docs/docs/zh/about/hardware.md +++ b/docs/docs/zh/about/hardware.md @@ -47,4 +47,5 @@ DolphinScheduler正常运行提供如下的网络端口配置： DolphinScheduler 推荐 Chrome 以及使用 Chromium 内核的较新版本浏览器访问前端可视化操作界面 ## 时钟同步 -为避免可能影响任务执行的内部集群通信问题，请确保所有集群节点上的时钟与公共时钟源同步，例如使用 Chrony 和/或 NTP。 同步时间确保集群中的每个节点都有相同的时间 \ No newline at end of file + +为避免可能影响任务执行的内部集群通信问题，请确保所有集群节点上的时钟与公共时钟源同步，例如使用 Chrony 和/或 NTP。 同步时间确保集群中的每个节点都有相同的时间 diff --git a/docs/docs/zh/contribute/join/become-a-committer.md b/docs/docs/zh/contribute/join/become-a-committer.md index 720b31d6f1..a1b5849972 100644 --- a/docs/docs/zh/contribute/join/become-a-committer.md +++ b/docs/docs/zh/contribute/join/become-a-committer.md @@ -28,6 +28,7 @@ PMC会定期基于活跃贡献者们对Dolphinscheduler的贡献从他们中提 发起提名的PMC成员负责向新的committer发出邀请，并指导他/她建立ASF精神思想。 PMC成员应使用以下模板向新的committer发送一封电子邮件: + ``` To: @gmail.com Cc: private@dolphinscheduler.apache.org diff --git a/docs/docs/zh/guide/datasource/oracle.md b/docs/docs/zh/guide/datasource/oracle.md index 581b0d8dd8..de5f07086f 100644 --- a/docs/docs/zh/guide/datasource/oracle.md +++ b/docs/docs/zh/guide/datasource/oracle.md @@ -12,3 +12,4 @@ - 数据库名：输入连接 ORACLE 的 ServiceName 或 SID - Jdbc 连接参数：用于 ORACLE 连接的参数设置，以 JSON 形式填写 - 比如 `{"schema": "abc"}` 指定使用 `username` 用户下的 `abc` 数据库 + diff --git a/docs/docs/zh/guide/parameter/built-in.md b/docs/docs/zh/guide/parameter/built-in.md index 4838c382d9..abf5ade789 100644 --- a/docs/docs/zh/guide/parameter/built-in.md +++ b/docs/docs/zh/guide/parameter/built-in.md @@ -29,33 +29,34 @@ - 也可以通过以下两种方式： - 1.使用add_months()函数，该函数用于加减月份， - 第一个入口参数为[yyyyMMdd]，表示返回时间的格式 - 第二个入口参数为月份偏移量，表示加减多少个月 - * 后 N 年：$[add_months(yyyyMMdd,12*N)] - * 前 N 年：$[add_months(yyyyMMdd,-12*N)] - * 后 N 月：$[add_months(yyyyMMdd,N)] - * 前 N 月：$[add_months(yyyyMMdd,-N)] - ******************************************* - 2.直接加减数字 - 在自定义格式后直接“+/-”数字 - * 后 N 周：$[yyyyMMdd+7*N] - * 前 N 周：$[yyyyMMdd-7*N] - * 后 N 天：$[yyyyMMdd+N] - * 前 N 天：$[yyyyMMdd-N] - * 后 N 小时：$[HHmmss+N/24] - * 前 N 小时：$[HHmmss-N/24] - * 后 N 分钟：$[HHmmss+N/24/60] - * 前 N 分钟：$[HHmmss-N/24/60] - ******************************************* - 3.业务属性方式 - 在自定义格式后直接“+/-”数字 - 支持日志格式：所有日期表达式，例如：yyyy-MM-dd/yyyyMMddHHmmss - * 当天：$[this_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-26 - * 昨天：$[last_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-25 - * 年的第N周，以周一为起点：$[year_week(yyyy-MM-dd)]，如：2022-08-26 => 2022-34 - * 年的第N周，以周N为起点：$[year_week(yyyy-MM-dd,N)]，如：N=5时 2022-08-26 => 2022-35 - * 前(-)/后(+) N 月第一天：$[month_first_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-26 => 2022-07-01 - * 前(-)/后(+) N 月最后一天：$[month_last_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-28 => 2022-07-31 - * 前(-)/后(+) N 周的周一：$[week_first_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-15 - * 前(-)/后(+) N 周的周日：$[week_last_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-21 + 1.使用add_months()函数，该函数用于加减月份， + 第一个入口参数为[yyyyMMdd]，表示返回时间的格式 + 第二个入口参数为月份偏移量，表示加减多少个月 + * 后 N 年：$[add_months(yyyyMMdd,12*N)] + * 前 N 年：$[add_months(yyyyMMdd,-12*N)] + * 后 N 月：$[add_months(yyyyMMdd,N)] + * 前 N 月：$[add_months(yyyyMMdd,-N)] + ******************************************* + 2.直接加减数字 + 在自定义格式后直接“+/-”数字 + * 后 N 周：$[yyyyMMdd+7*N] + * 前 N 周：$[yyyyMMdd-7*N] + * 后 N 天：$[yyyyMMdd+N] + * 前 N 天：$[yyyyMMdd-N] + * 后 N 小时：$[HHmmss+N/24] + * 前 N 小时：$[HHmmss-N/24] + * 后 N 分钟：$[HHmmss+N/24/60] + * 前 N 分钟：$[HHmmss-N/24/60] + ******************************************* + 3.业务属性方式 + 在自定义格式后直接“+/-”数字 + 支持日志格式：所有日期表达式，例如：yyyy-MM-dd/yyyyMMddHHmmss + * 当天：$[this_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-26 + * 昨天：$[last_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-25 + * 年的第N周，以周一为起点：$[year_week(yyyy-MM-dd)]，如：2022-08-26 => 2022-34 + * 年的第N周，以周N为起点：$[year_week(yyyy-MM-dd,N)]，如：N=5时 2022-08-26 => 2022-35 + * 前(-)/后(+) N 月第一天：$[month_first_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-26 => 2022-07-01 + * 前(-)/后(+) N 月最后一天：$[month_last_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-28 => 2022-07-31 + * 前(-)/后(+) N 周的周一：$[week_first_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-15 + * 前(-)/后(+) N 周的周日：$[week_last_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-21 + diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 0bc0a7e675..4e294f954d 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -11,3 +11,4 @@ * Copy and import workflow without 'copy' suffix [#10607](https://github.com/apache/dolphinscheduler/pull/10607) * Use semicolon as default sql segment separator [#10869](https://github.com/apache/dolphinscheduler/pull/10869) + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml index 45ec526859..9719689f39 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml @@ -17,22 +17,21 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 - dolphinscheduler-alert org.apache.dolphinscheduler + dolphinscheduler-alert dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-api org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-spi + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java index b461555a0e..530a548342 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java @@ -25,6 +25,7 @@ import lombok.NonNull; * alert channel for sending alerts */ public interface AlertChannel { + /** * process and send alert * diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java index 0f9878485c..55097ee3af 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java @@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.alert.api; import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; +import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import java.util.List; @@ -29,6 +29,7 @@ import java.util.List; * alert channel factory */ public interface AlertChannelFactory extends PrioritySPI { + /** * Returns the name of the alert channel * diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java index 6c0848fc9d..a94a31a8a9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.alert.api; public final class AlertConstants { + /** * the field name of alert show type **/ diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java index 95dccae7eb..46b87320c6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java @@ -25,6 +25,7 @@ import lombok.Getter; @AllArgsConstructor @Getter public enum ShowType { + /** * 0 TABLE; * 1 TEXT; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml index 34b1ef3335..5ce2f6d9d1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-dingtalk jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java index 235f61c4d3..bfa694cd28 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java @@ -38,6 +38,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class DingTalkAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "DingTalk"; @@ -67,8 +68,10 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory { RadioParam msgTypeParam = RadioParam .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE, DingTalkParamsConstants.DING_TALK_MSG_TYPE) - .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, false)) - .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, false)) + .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, + DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, false)) + .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, + DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, false)) .setValue(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT) .addValidate(Validate.newBuilder() .setRequired(false) @@ -76,13 +79,15 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory { .build(); InputParam atMobilesParam = InputParam - .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES, DingTalkParamsConstants.DING_TALK_AT_MOBILES) + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES, + DingTalkParamsConstants.DING_TALK_AT_MOBILES) .addValidate(Validate.newBuilder() .setRequired(false) .build()) .build(); InputParam atUserIdsParam = InputParam - .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS, DingTalkParamsConstants.DING_TALK_AT_USERIDS) + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS, + DingTalkParamsConstants.DING_TALK_AT_USERIDS) .addValidate(Validate.newBuilder() .setRequired(false) .build()) @@ -98,7 +103,8 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory { .build(); RadioParam isEnableProxy = RadioParam - .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, DingTalkParamsConstants.DING_TALK_PROXY_ENABLE) + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, + DingTalkParamsConstants.DING_TALK_PROXY_ENABLE) .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) .setValue(STRING_FALSE) @@ -131,7 +137,8 @@ public final class DingTalkAlertChannelFactory implements AlertChannelFactory { .setPlaceholder("if enable use authentication, you need input password") .build(); - return Arrays.asList(webHookParam, keywordParam, secretParam, msgTypeParam, atMobilesParam, atUserIdsParam, isAtAll, isEnableProxy, proxyParam, portParam, userParam, passwordParam); + return Arrays.asList(webHookParam, keywordParam, secretParam, msgTypeParam, atMobilesParam, atUserIdsParam, + isAtAll, isEnableProxy, proxyParam, portParam, userParam, passwordParam); } @Override diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java index e5b667ae30..d68a1bde52 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk; public final class DingTalkParamsConstants { + static final String DING_TALK_PROXY_ENABLE = "$t('isEnableProxy')"; static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index f8a824aec8..617d051794 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -171,7 +171,8 @@ public final class DingTalkSender { String msg = generateMsgJson(title, content); - HttpPost httpPost = constructHttpPost(org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msg); + HttpPost httpPost = constructHttpPost( + org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msg); CloseableHttpClient httpClient; if (Boolean.TRUE.equals(enableProxy)) { @@ -289,8 +290,12 @@ public final class DingTalkSender { private void setMsgAt(Map items) { Map at = new HashMap<>(); - String[] atMobileArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",") : new String[0]; - String[] atUserArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",") : new String[0]; + String[] atMobileArray = + org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",") + : new String[0]; + String[] atUserArray = + org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",") + : new String[0]; boolean isAtAll = Objects.isNull(atAll) ? false : atAll; at.put("atMobiles", atMobileArray); @@ -313,7 +318,7 @@ public final class DingTalkSender { Mac mac = Mac.getInstance("HmacSHA256"); mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256")); byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8")); - sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)),"UTF-8"); + sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8"); } catch (Exception e) { logger.error("generate sign error, message:{}", e); } @@ -321,6 +326,7 @@ public final class DingTalkSender { } static final class DingTalkSendMsgResponse { + private Integer errcode; private String errmsg; @@ -378,7 +384,8 @@ public final class DingTalkSender { @Override public String toString() { - return "DingTalkSender.DingTalkSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + this.getErrmsg() + ")"; + return "DingTalkSender.DingTalkSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + + this.getErrmsg() + ")"; } } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml index 1ec2302022..4669d7eac0 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-email jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java index 3c29ca24f5..663d251ea3 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java @@ -28,6 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class EmailAlertChannel implements AlertChannel { + private static final Logger logger = LoggerFactory.getLogger(EmailAlertChannel.class); @Override diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java index 2f5d6aae28..8a38cfa370 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java @@ -40,6 +40,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class EmailAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Email"; @@ -48,73 +49,88 @@ public final class EmailAlertChannelFactory implements AlertChannelFactory { @Override public List params() { List paramsList = new ArrayList<>(); - InputParam receivesParam = InputParam.newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERS) - .setPlaceholder("please input receives") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); - - InputParam receiveCcsParam = InputParam.newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERCCS) - .build(); - - InputParam mailSmtpHost = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_HOST, MailParamsConstants.MAIL_SMTP_HOST) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - InputParam mailSmtpPort = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_PORT, MailParamsConstants.MAIL_SMTP_PORT) - .setValue("25") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); - - InputParam mailSender = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SENDER, MailParamsConstants.MAIL_SENDER) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - RadioParam enableSmtpAuth = RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_AUTH, MailParamsConstants.MAIL_SMTP_AUTH) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_TRUE) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + InputParam receivesParam = InputParam + .newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, + MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERS) + .setPlaceholder("please input receives") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); + + InputParam receiveCcsParam = InputParam + .newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, + MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERCCS) + .build(); + + InputParam mailSmtpHost = + InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_HOST, MailParamsConstants.MAIL_SMTP_HOST) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + InputParam mailSmtpPort = + InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_PORT, MailParamsConstants.MAIL_SMTP_PORT) + .setValue("25") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); + + InputParam mailSender = + InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SENDER, MailParamsConstants.MAIL_SENDER) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + RadioParam enableSmtpAuth = + RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_AUTH, MailParamsConstants.MAIL_SMTP_AUTH) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_TRUE) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); InputParam mailUser = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_USER, MailParamsConstants.MAIL_USER) - .setPlaceholder("if enable use authentication, you need input user") - .build(); - - PasswordParam mailPassword = PasswordParam.newBuilder(MailParamsConstants.NAME_MAIL_PASSWD, MailParamsConstants.MAIL_PASSWD) - .setPlaceholder("if enable use authentication, you need input password") - .build(); - - RadioParam enableTls = RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, MailParamsConstants.MAIL_SMTP_STARTTLS_ENABLE) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_FALSE) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - RadioParam enableSsl = RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, MailParamsConstants.MAIL_SMTP_SSL_ENABLE) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_FALSE) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - InputParam sslTrust = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, MailParamsConstants.MAIL_SMTP_SSL_TRUST) - .setValue("*") - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + .setPlaceholder("if enable use authentication, you need input user") + .build(); + + PasswordParam mailPassword = + PasswordParam.newBuilder(MailParamsConstants.NAME_MAIL_PASSWD, MailParamsConstants.MAIL_PASSWD) + .setPlaceholder("if enable use authentication, you need input password") + .build(); + + RadioParam enableTls = RadioParam + .newBuilder(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, + MailParamsConstants.MAIL_SMTP_STARTTLS_ENABLE) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_FALSE) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + RadioParam enableSsl = RadioParam + .newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, MailParamsConstants.MAIL_SMTP_SSL_ENABLE) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_FALSE) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + InputParam sslTrust = InputParam + .newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, MailParamsConstants.MAIL_SMTP_SSL_TRUST) + .setValue("*") + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); RadioParam showType = RadioParam.newBuilder(AlertConstants.NAME_SHOW_TYPE, AlertConstants.SHOW_TYPE) - .addParamsOptions(new ParamsOptions(ShowType.TABLE.getDescp(), ShowType.TABLE.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.ATTACHMENT.getDescp(), ShowType.ATTACHMENT.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.TABLE_ATTACHMENT.getDescp(), ShowType.TABLE_ATTACHMENT.getDescp(), false)) - .setValue(ShowType.TABLE.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + .addParamsOptions(new ParamsOptions(ShowType.TABLE.getDescp(), ShowType.TABLE.getDescp(), false)) + .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) + .addParamsOptions( + new ParamsOptions(ShowType.ATTACHMENT.getDescp(), ShowType.ATTACHMENT.getDescp(), false)) + .addParamsOptions(new ParamsOptions(ShowType.TABLE_ATTACHMENT.getDescp(), + ShowType.TABLE_ATTACHMENT.getDescp(), false)) + .setValue(ShowType.TABLE.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); paramsList.add(receivesParam); paramsList.add(receiveCcsParam); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java index 4c6e1d37ee..94eb4efa39 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.email; public final class EmailConstants { + public static final String XLS_FILE_PATH = "xls.file.path"; public static final String MAIL_TRANSPORT_PROTOCOL = "mail.transport.protocol"; @@ -38,19 +39,20 @@ public final class EmailConstants { public static final String TH_END = ""; - public static final String HTML_HEADER_PREFIX = "" - + "" - + "" - + "dolphinscheduler" - + "" - + "" - + "" - + "" - + " "; + public static final String HTML_HEADER_PREFIX = + "" + + "" + + "" + + "dolphinscheduler" + + "" + + "" + + "" + + "" + + "
"; public static final String TABLE_BODY_HTML_TAIL = "
"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java index 6e02178f19..fbb22cde65 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class ExcelUtils { + private static final int XLSX_WINDOW_ROW = 10000; private static final Logger logger = LoggerFactory.getLogger(ExcelUtils.class); @@ -74,32 +75,33 @@ public final class ExcelUtils { for (Map.Entry en : headerMap.entrySet()) { headerList.add(en.getKey()); } - try (SXSSFWorkbook wb = new SXSSFWorkbook(XLSX_WINDOW_ROW); - FileOutputStream fos = new FileOutputStream(String.format("%s/%s.xlsx", xlsFilePath, title))) { + try ( + SXSSFWorkbook wb = new SXSSFWorkbook(XLSX_WINDOW_ROW); + FileOutputStream fos = new FileOutputStream(String.format("%s/%s.xlsx", xlsFilePath, title))) { // declare a workbook // generate a table Sheet sheet = wb.createSheet(); Row row = sheet.createRow(0); - //set the height of the first line + // set the height of the first line row.setHeight((short) 500); - //set Horizontal right + // set Horizontal right CellStyle cellStyle = wb.createCellStyle(); cellStyle.setAlignment(HorizontalAlignment.RIGHT); - //setting excel headers + // setting excel headers for (int i = 0; i < headerList.size(); i++) { Cell cell = row.createCell(i); cell.setCellStyle(cellStyle); cell.setCellValue(headerList.get(i)); } - //setting excel body + // setting excel body int rowIndex = 1; for (LinkedHashMap itemsMap : itemsList) { Object[] values = itemsMap.values().toArray(); row = sheet.createRow(rowIndex); - //setting excel body height + // setting excel body height row.setHeight((short) 500); rowIndex++; for (int j = 0; j < values.length; j++) { @@ -117,7 +119,7 @@ public final class ExcelUtils { sheet.setColumnWidth(i, headerList.get(i).length() * 800); } - //setting file output + // setting file output wb.write(fos); wb.dispose(); } catch (Exception e) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java index bf84fd634d..5fa8c36da3 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.email; public final class MailParamsConstants { + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERS = "$t('receivers')"; public static final String NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS = "receivers"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index a7305cf741..5f805f14ad 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -60,6 +60,7 @@ import org.slf4j.LoggerFactory; import com.sun.mail.smtp.SMTPProvider; public final class MailSender { + private static final Logger logger = LoggerFactory.getLogger(MailSender.class); private final List receivers; @@ -178,7 +179,7 @@ public final class MailSender { } if (CollectionUtils.isNotEmpty(receiverCcs)) { - //cc + // cc for (String receiverCc : receiverCcs) { email.addCc(receiverCc); } @@ -188,12 +189,13 @@ public final class MailSender { } catch (Exception e) { handleException(alertResult, e); } - } else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLE_ATTACHMENT.getDescp())) { + } else if (showType.equals(ShowType.ATTACHMENT.getDescp()) + || showType.equals(ShowType.TABLE_ATTACHMENT.getDescp())) { try { String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) - ? "Please see the attachment " + title + EmailConstants.EXCEL_SUFFIX_XLSX - : htmlTable(content, false)); + ? "Please see the attachment " + title + EmailConstants.EXCEL_SUFFIX_XLSX + : htmlTable(content, false)); attachment(title, content, partContent); @@ -294,6 +296,7 @@ public final class MailSender { props.setProperty(MailParamsConstants.MAIL_SMTP_SSL_TRUST, sslTrust); Authenticator auth = new Authenticator() { + @Override protected PasswordAuthentication getPasswordAuthentication() { // mail username and password @@ -309,7 +312,8 @@ public final class MailSender { /** * attach content */ - private void attachContent(String title, String content, String partContent, MimeMessage msg) throws MessagingException, IOException { + private void attachContent(String title, String content, String partContent, + MimeMessage msg) throws MessagingException, IOException { /* * set receiverCc */ @@ -329,7 +333,8 @@ public final class MailSender { MimeBodyPart part2 = new MimeBodyPart(); // add random uuid to filename to avoid potential issue String randomFilename = title + UUID.randomUUID(); - File file = new File(xlsFilePath + EmailConstants.SINGLE_SLASH + randomFilename + EmailConstants.EXCEL_SUFFIX_XLSX); + File file = + new File(xlsFilePath + EmailConstants.SINGLE_SLASH + randomFilename + EmailConstants.EXCEL_SUFFIX_XLSX); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } @@ -352,7 +357,8 @@ public final class MailSender { /** * the string object map */ - private AlertResult getStringObjectMap(String title, String content, AlertResult alertResult, HtmlEmail email) throws EmailException { + private AlertResult getStringObjectMap(String title, String content, AlertResult alertResult, + HtmlEmail email) throws EmailException { /* * the subject of the message to be sent diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java index 5b64d8de1e..6b45771692 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.email.exception; public class AlertEmailException extends RuntimeException { + public AlertEmailException(String errMsg) { super(errMsg); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java index 4167e77235..8d2f1ab14e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java @@ -33,15 +33,14 @@ import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.boot.configurationprocessor.json.JSONArray; +import org.springframework.boot.configurationprocessor.json.JSONException; +import org.springframework.boot.configurationprocessor.json.JSONTokener; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.springframework.boot.configurationprocessor.json.JSONArray; -import org.springframework.boot.configurationprocessor.json.JSONException; -import org.springframework.boot.configurationprocessor.json.JSONTokener; - public class DefaultHTMLTemplate implements AlertTemplate { public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class); @@ -55,7 +54,8 @@ public class DefaultHTMLTemplate implements AlertTemplate { case TEXT: return getTextTypeMessage(content); default: - throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate", showType)); + throw new IllegalArgumentException( + String.format("not support showType: %s in DefaultHTMLTemplate", showType)); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml index 93450e08f6..1a56cdc094 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-feishu jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java index 509eebe615..8959c8aaec 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class FeiShuAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java index da78b9f39b..7f6e65ca93 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java @@ -38,6 +38,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class FeiShuAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Feishu"; @@ -45,39 +46,44 @@ public final class FeiShuAlertChannelFactory implements AlertChannelFactory { @Override public List params() { - InputParam webHookParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_WEB_HOOK, FeiShuParamsConstants.WEB_HOOK) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam webHookParam = + InputParam.newBuilder(FeiShuParamsConstants.NAME_WEB_HOOK, FeiShuParamsConstants.WEB_HOOK) + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); RadioParam isEnableProxy = - RadioParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE, FeiShuParamsConstants.FEI_SHU_PROXY_ENABLE) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_TRUE) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + RadioParam + .newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE, + FeiShuParamsConstants.FEI_SHU_PROXY_ENABLE) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_TRUE) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); InputParam proxyParam = - InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY, FeiShuParamsConstants.FEI_SHU_PROXY) - .addValidate(Validate.newBuilder() - .setRequired(false).build()) - .build(); + InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY, FeiShuParamsConstants.FEI_SHU_PROXY) + .addValidate(Validate.newBuilder() + .setRequired(false).build()) + .build(); - InputParam portParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PORT, FeiShuParamsConstants.FEI_SHU_PORT) - .addValidate(Validate.newBuilder() - .setRequired(false).build()) - .build(); + InputParam portParam = + InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PORT, FeiShuParamsConstants.FEI_SHU_PORT) + .addValidate(Validate.newBuilder() + .setRequired(false).build()) + .build(); InputParam userParam = - InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_USER, FeiShuParamsConstants.FEI_SHU_USER) - .addValidate(Validate.newBuilder() - .setRequired(false).build()) - .build(); - PasswordParam passwordParam = PasswordParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD, FeiShuParamsConstants.FEI_SHU_PASSWORD) - .setPlaceholder("if enable use authentication, you need input password") - .build(); + InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_USER, FeiShuParamsConstants.FEI_SHU_USER) + .addValidate(Validate.newBuilder() + .setRequired(false).build()) + .build(); + PasswordParam passwordParam = PasswordParam + .newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD, FeiShuParamsConstants.FEI_SHU_PASSWORD) + .setPlaceholder("if enable use authentication, you need input password") + .build(); return Arrays.asList(webHookParam, isEnableProxy, proxyParam, portParam, userParam, passwordParam); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java index aa743903e9..0d2c3912f8 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.feishu; public final class FeiShuParamsConstants { + public static final String FEI_SHU_PROXY_ENABLE = "$t('isEnableProxy')"; static final String WEB_HOOK = "$t('webhook')"; static final String NAME_WEB_HOOK = "WebHook"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index a5d11c3684..ba245c3ae2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonProperty; public final class FeiShuSender { + private static final Logger logger = LoggerFactory.getLogger(FeiShuSender.class); private final String url; private final Boolean enableProxy; @@ -98,7 +99,8 @@ public final class FeiShuSender { return alertResult; } alertResult.setMessage(String.format("alert send fei shu msg error : %s", sendMsgResponse.getStatusMessage())); - logger.info("alert send fei shu msg error : {} ,Extra : {} ", sendMsgResponse.getStatusMessage(), sendMsgResponse.getExtra()); + logger.info("alert send fei shu msg error : {} ,Extra : {} ", sendMsgResponse.getStatusMessage(), + sendMsgResponse.getExtra()); return alertResult; } @@ -164,7 +166,8 @@ public final class FeiShuSender { } finally { response.close(); } - logger.info("Fei Shu send title :{} ,content :{}, resp: {}", alertData.getTitle(), alertData.getContent(), resp); + logger.info("Fei Shu send title :{} ,content :{}, resp: {}", alertData.getTitle(), alertData.getContent(), + resp); return resp; } finally { httpClient.close(); @@ -172,6 +175,7 @@ public final class FeiShuSender { } static final class FeiShuSendMsgResponse { + @JsonProperty("Extra") private String extra; @JsonProperty("StatusCode") @@ -229,7 +233,8 @@ public final class FeiShuSender { } final Object this$statusMessage = this.getStatusMessage(); final Object other$statusMessage = other.getStatusMessage(); - if (this$statusMessage == null ? other$statusMessage != null : !this$statusMessage.equals(other$statusMessage)) { + if (this$statusMessage == null ? other$statusMessage != null + : !this$statusMessage.equals(other$statusMessage)) { return false; } return true; @@ -248,7 +253,8 @@ public final class FeiShuSender { } public String toString() { - return "FeiShuSender.FeiShuSendMsgResponse(extra=" + this.getExtra() + ", statusCode=" + this.getStatusCode() + ", statusMessage=" + this.getStatusMessage() + ")"; + return "FeiShuSender.FeiShuSendMsgResponse(extra=" + this.getExtra() + ", statusCode=" + + this.getStatusCode() + ", statusMessage=" + this.getStatusMessage() + ")"; } } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java index eca9b07d15..2ba373df08 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java @@ -29,11 +29,13 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; public final class HttpRequestUtil { + private HttpRequestUtil() { throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); } - public static CloseableHttpClient getHttpClient(boolean enableProxy, String proxy, Integer port, String user, String password) { + public static CloseableHttpClient getHttpClient(boolean enableProxy, String proxy, Integer port, String user, + String password) { if (enableProxy) { HttpHost httpProxy = new HttpHost(proxy, port); CredentialsProvider provider = new BasicCredentialsProvider(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml index 007c3cd835..c83356af4d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-http jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java index 14b416785d..944762f13f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class HttpAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java index c18ebe4454..0a54b42357 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java @@ -30,6 +30,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class HttpAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Http"; @@ -39,39 +40,43 @@ public final class HttpAlertChannelFactory implements AlertChannelFactory { public List params() { InputParam url = InputParam.newBuilder(HttpAlertConstants.NAME_URL, HttpAlertConstants.URL) - .setPlaceholder("input request URL") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + .setPlaceholder("input request URL") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam headerParams = InputParam.newBuilder(HttpAlertConstants.NAME_HEADER_PARAMS, HttpAlertConstants.HEADER_PARAMS) - .setPlaceholder("input request headers as JSON format ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam headerParams = + InputParam.newBuilder(HttpAlertConstants.NAME_HEADER_PARAMS, HttpAlertConstants.HEADER_PARAMS) + .setPlaceholder("input request headers as JSON format ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam bodyParams = InputParam.newBuilder(HttpAlertConstants.NAME_BODY_PARAMS, HttpAlertConstants.BODY_PARAMS) - .setPlaceholder("input request body as JSON format ") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam bodyParams = + InputParam.newBuilder(HttpAlertConstants.NAME_BODY_PARAMS, HttpAlertConstants.BODY_PARAMS) + .setPlaceholder("input request body as JSON format ") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam contentField = InputParam.newBuilder(HttpAlertConstants.NAME_CONTENT_FIELD, HttpAlertConstants.CONTENT_FIELD) - .setPlaceholder("input alert msg field name") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam contentField = + InputParam.newBuilder(HttpAlertConstants.NAME_CONTENT_FIELD, HttpAlertConstants.CONTENT_FIELD) + .setPlaceholder("input alert msg field name") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam requestType = InputParam.newBuilder(HttpAlertConstants.NAME_REQUEST_TYPE, HttpAlertConstants.REQUEST_TYPE) - .setPlaceholder("input request type POST or GET") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam requestType = + InputParam.newBuilder(HttpAlertConstants.NAME_REQUEST_TYPE, HttpAlertConstants.REQUEST_TYPE) + .setPlaceholder("input request type POST or GET") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); return Arrays.asList(url, requestType, headerParams, bodyParams, contentField); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java index 6bbf0bfbe0..22db292d5f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.http; public final class HttpAlertConstants { + public static final String URL = "$t('url')"; public static final String NAME_URL = "url"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml index 08a664d531..671bff25ea 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-pagerduty jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java index 6ad4c296c1..b033139520 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class PagerDutyAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java index 24840f730e..92c42b9d04 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java @@ -17,17 +17,20 @@ package org.apache.dolphinscheduler.plugin.alert.pagerduty; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertChannelFactory; import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.input.InputParam; + import java.util.Collections; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(AlertChannelFactory.class) public final class PagerDutyAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "PagerDuty"; @@ -35,7 +38,9 @@ public final class PagerDutyAlertChannelFactory implements AlertChannelFactory { @Override public List params() { - InputParam integrationKey = InputParam.newBuilder(PagerDutyParamsConstants.NAME_PAGER_DUTY_INTEGRATION_KEY_NAME, PagerDutyParamsConstants.PAGER_DUTY_INTEGRATION_KEY) + InputParam integrationKey = InputParam + .newBuilder(PagerDutyParamsConstants.NAME_PAGER_DUTY_INTEGRATION_KEY_NAME, + PagerDutyParamsConstants.PAGER_DUTY_INTEGRATION_KEY) .addValidate(Validate.newBuilder() .setRequired(true) .build()) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java index a90d302504..bb9ccf6a3e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.pagerduty; public final class PagerDutyParamsConstants { + public static final String NAME_PAGER_DUTY_INTEGRATION_KEY_NAME = "IntegrationKey"; public static final String PAGER_DUTY_INTEGRATION_KEY = "integrationKey"; public static final String PAGER_DUTY_EVENT_ACTION_TRIGGER = "trigger"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java index 3dd0b840c0..5fe11e00a7 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java @@ -17,21 +17,29 @@ package org.apache.dolphinscheduler.plugin.alert.pagerduty; -import com.google.common.base.Preconditions; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.spi.utils.JSONUtils; + import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.slf4j.Logger; + import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.slf4j.Logger; + +import com.google.common.base.Preconditions; public final class PagerDutySender { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(PagerDutySender.class); private final String integrationKey; @@ -41,7 +49,7 @@ public final class PagerDutySender { Preconditions.checkArgument(!Objects.isNull(integrationKey), "PagerDuty integration key can not be null"); } - public AlertResult sendPagerDutyAlter(String title, String content){ + public AlertResult sendPagerDutyAlter(String title, String content) { AlertResult alertResult = new AlertResult(); alertResult.setStatus("false"); alertResult.setMessage("send pager duty alert fail."); @@ -72,10 +80,10 @@ public final class PagerDutySender { if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_ACCEPTED) { alertResult.setStatus("true"); alertResult.setMessage("send pager duty alert success"); - }else { + } else { log.info("send pager duty alert fail, statusCode : {}", statusCode); } - }finally { + } finally { response.close(); } } catch (IOException e) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml index 96038b970c..e5b168d543 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-script jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java index 51e567b47e..ef5fcf67d2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.script; public final class OSUtils { + private OSUtils() { throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java index 0bc0a121d6..6ea9a8db92 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class ProcessUtils { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); private ProcessUtils() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java index bd52955b74..3d865c6dc0 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class ScriptAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertinfo) { AlertData alertData = alertinfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java index aacde95ba1..294bca04fd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java @@ -32,6 +32,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class ScriptAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Script"; @@ -40,26 +41,30 @@ public final class ScriptAlertChannelFactory implements AlertChannelFactory { @Override public List params() { - InputParam scriptUserParam = InputParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, ScriptParamsConstants.SCRIPT_USER_PARAMS) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .setPlaceholder("please enter your custom parameters, which will be passed to you when calling your script") - .build(); + InputParam scriptUserParam = InputParam + .newBuilder(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, ScriptParamsConstants.SCRIPT_USER_PARAMS) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .setPlaceholder( + "please enter your custom parameters, which will be passed to you when calling your script") + .build(); // need check file type and file exist - InputParam scriptPathParam = InputParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_PATH, ScriptParamsConstants.SCRIPT_PATH) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .setPlaceholder("please upload the file to the disk directory of the alert server," - + " and ensure that the path is absolute and has the corresponding access rights") - .build(); + InputParam scriptPathParam = + InputParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_PATH, ScriptParamsConstants.SCRIPT_PATH) + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .setPlaceholder("please upload the file to the disk directory of the alert server," + + " and ensure that the path is absolute and has the corresponding access rights") + .build(); - RadioParam scriptTypeParams = RadioParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_TYPE, ScriptParamsConstants.SCRIPT_TYPE) - .addParamsOptions(new ParamsOptions(ScriptType.SHELL.getDescp(), ScriptType.SHELL.getDescp(), false)) - .setValue(ScriptType.SHELL.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + RadioParam scriptTypeParams = RadioParam + .newBuilder(ScriptParamsConstants.NAME_SCRIPT_TYPE, ScriptParamsConstants.SCRIPT_TYPE) + .addParamsOptions(new ParamsOptions(ScriptType.SHELL.getDescp(), ScriptType.SHELL.getDescp(), false)) + .setValue(ScriptType.SHELL.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); return Arrays.asList(scriptUserParam, scriptPathParam, scriptTypeParams); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java index e680903fb2..1c759525b3 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.script; public final class ScriptParamsConstants { + static final String SCRIPT_TYPE = "$t('scriptType')"; static final String NAME_SCRIPT_TYPE = "type"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java index cbb3e11f28..b31e10f575 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.script; public enum ScriptType { + SHELL(0, "SHELL"), ; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java index 208791b6f2..e2765a8a40 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class StreamGobbler extends Thread { + private static final Logger logger = LoggerFactory.getLogger(StreamGobbler.class); private final InputStream inputStream; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml index 256b1ebd67..e92ddc6a42 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-slack jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java index b63fed0574..c8cb36a78b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class SlackAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java index f86e32933f..e08bc4618e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java @@ -30,6 +30,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class SlackAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Slack"; @@ -39,19 +40,20 @@ public final class SlackAlertChannelFactory implements AlertChannelFactory { public List params() { List paramsList = new LinkedList<>(); - InputParam webHookParam = InputParam.newBuilder(SlackParamsConstants.SLACK_WEB_HOOK_URL_NAME, SlackParamsConstants.SLACK_WEB_HOOK_URL) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .setPlaceholder("Input WebHook Url") - .build(); + InputParam webHookParam = InputParam + .newBuilder(SlackParamsConstants.SLACK_WEB_HOOK_URL_NAME, SlackParamsConstants.SLACK_WEB_HOOK_URL) + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .setPlaceholder("Input WebHook Url") + .build(); InputParam botName = InputParam.newBuilder(SlackParamsConstants.SLACK_BOT_NAME, SlackParamsConstants.SLACK_BOT) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .setPlaceholder("Input the bot username") - .build(); + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .setPlaceholder("Input the bot username") + .build(); paramsList.add(webHookParam); paramsList.add(botName); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java index a3c9f19ac5..c92f303155 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.slack; public final class SlackParamsConstants { + public static final String SLACK_WEB_HOOK_URL = "$t('webhook')"; public static final String SLACK_WEB_HOOK_URL_NAME = "webHook"; public static final String SLACK_BOT = "$t('Username')"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java index 1017c839fd..e571642f38 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; public final class SlackSender { + private static final Logger logger = LoggerFactory.getLogger(SlackSender.class); private final String webHookUrl; @@ -53,7 +54,8 @@ public final class SlackSender { webHookUrl = slackAlertParam.get(SlackParamsConstants.SLACK_WEB_HOOK_URL_NAME); botName = slackAlertParam.get(SlackParamsConstants.SLACK_BOT_NAME); Preconditions.checkArgument(!Objects.isNull(webHookUrl), "SlackWebHookURL can not be null"); - Preconditions.checkArgument(webHookUrl.startsWith("https://hooks.slack.com/services/"), "SlackWebHookURL invalidate"); + Preconditions.checkArgument(webHookUrl.startsWith("https://hooks.slack.com/services/"), + "SlackWebHookURL invalidate"); Preconditions.checkArgument(!Objects.isNull(botName), "slack bot name can not be null"); } @@ -120,17 +122,17 @@ public final class SlackSender { final int elementLen = maxLen; StringBuilder stringBuilder = new StringBuilder(200); stringBuilder.append(headers.stream() - .map(header -> generateString(header, elementLen, " ")) - .collect(Collectors.joining("|"))); + .map(header -> generateString(header, elementLen, " ")) + .collect(Collectors.joining("|"))); stringBuilder.append("\n"); for (List element : elements) { stringBuilder.append(element.stream() - .map(lement -> generateString("", elementLen, "-")) - .collect(Collectors.joining("|"))); + .map(lement -> generateString("", elementLen, "-")) + .collect(Collectors.joining("|"))); stringBuilder.append("\n"); stringBuilder.append(element.stream() - .map(e -> generateString(e, elementLen, " ")) - .collect(Collectors.joining("|"))); + .map(e -> generateString(e, elementLen, " ")) + .collect(Collectors.joining("|"))); stringBuilder.append("\n"); } return String.format("```%s```", stringBuilder); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml index ed761d446a..e24045c8aa 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-telegram jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java index 26db453dd4..efc8912d1a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class TelegramAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo info) { Map alertParams = info.getAlertParams(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java index f282cb5153..240578cfb7 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java @@ -17,7 +17,11 @@ package org.apache.dolphinscheduler.plugin.alert.telegram; -import com.google.auto.service.AutoService; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_FALSE; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_NO; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_TRUE; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_YES; + import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertChannelFactory; import org.apache.dolphinscheduler.spi.params.PasswordParam; @@ -31,7 +35,7 @@ import org.apache.dolphinscheduler.spi.params.select.SelectParam; import java.util.Arrays; import java.util.List; -import static org.apache.dolphinscheduler.spi.utils.Constants.*; +import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class TelegramAlertChannelFactory implements AlertChannelFactory { @@ -49,40 +53,50 @@ public final class TelegramAlertChannelFactory implements AlertChannelFactory { @Override public List params() { - - InputParam webHookParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_WEB_HOOK, TelegramParamsConstants.TELEGRAM_WEB_HOOK) + InputParam webHookParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_WEB_HOOK, TelegramParamsConstants.TELEGRAM_WEB_HOOK) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .setPlaceholder("telegram web hook") .build(); - InputParam botTokenParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, TelegramParamsConstants.TELEGRAM_BOT_TOKEN) + InputParam botTokenParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, TelegramParamsConstants.TELEGRAM_BOT_TOKEN) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .setPlaceholder("telegram bot token") .build(); - InputParam chatIdParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, TelegramParamsConstants.TELEGRAM_CHAT_ID) + InputParam chatIdParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, TelegramParamsConstants.TELEGRAM_CHAT_ID) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .setPlaceholder("telegram channel chat id") .build(); - SelectParam parseMode = SelectParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramParamsConstants.TELEGRAM_PARSE_MODE) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_TXT, TelegramAlertConstants.PARSE_MODE_TXT, false)) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN, TelegramAlertConstants.PARSE_MODE_MARKDOWN, false)) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, false)) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_HTML, TelegramAlertConstants.PARSE_MODE_HTML, false)) + SelectParam parseMode = SelectParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, + TelegramParamsConstants.TELEGRAM_PARSE_MODE) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_TXT, + TelegramAlertConstants.PARSE_MODE_TXT, false)) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN, + TelegramAlertConstants.PARSE_MODE_MARKDOWN, false)) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, + TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, false)) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_HTML, + TelegramAlertConstants.PARSE_MODE_HTML, false)) .setValue(TelegramAlertConstants.PARSE_MODE_TXT) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .build(); - RadioParam isEnableProxy = RadioParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY_ENABLE, TelegramParamsConstants.TELEGRAM_PROXY_ENABLE) + RadioParam isEnableProxy = RadioParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY_ENABLE, + TelegramParamsConstants.TELEGRAM_PROXY_ENABLE) .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) .setValue(STRING_FALSE) @@ -91,33 +105,37 @@ public final class TelegramAlertChannelFactory implements AlertChannelFactory { .build()) .build(); - InputParam proxyParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY, TelegramParamsConstants.TELEGRAM_PROXY) + InputParam proxyParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY, TelegramParamsConstants.TELEGRAM_PROXY) .addValidate(Validate.newBuilder() .setRequired(false) .build()) .build(); - InputParam portParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PORT, TelegramParamsConstants.TELEGRAM_PORT) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); - - InputParam userParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_USER, TelegramParamsConstants.TELEGRAM_USER) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); - - PasswordParam passwordParam = PasswordParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PASSWORD, TelegramParamsConstants.TELEGRAM_PASSWORD) + InputParam portParam = + InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PORT, TelegramParamsConstants.TELEGRAM_PORT) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + + InputParam userParam = + InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_USER, TelegramParamsConstants.TELEGRAM_USER) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + + PasswordParam passwordParam = PasswordParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PASSWORD, TelegramParamsConstants.TELEGRAM_PASSWORD) .addValidate(Validate.newBuilder() .setRequired(false) .build()) .setPlaceholder("if enable use authentication, you need input password") .build(); - return Arrays.asList(webHookParam, botTokenParam, chatIdParam, parseMode, isEnableProxy, proxyParam, portParam, userParam, passwordParam); + return Arrays.asList(webHookParam, botTokenParam, chatIdParam, parseMode, isEnableProxy, proxyParam, portParam, + userParam, passwordParam); } - } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java index d9fbb3f218..4ae48506c4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.alert.telegram; - public final class TelegramParamsConstants { static final String TELEGRAM_WEB_HOOK = "$t('webHook')"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 2472e81a40..eb7d7bfa3c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.alert.telegram; -import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.dolphinscheduler.alert.api.AlertData; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; + import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; @@ -37,14 +37,17 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonProperty; + public final class TelegramSender { private static final Logger logger = LoggerFactory.getLogger(TelegramSender.class); @@ -69,7 +72,6 @@ public final class TelegramSender { private String password; - TelegramSender(Map config) { url = config.get(TelegramParamsConstants.NAME_TELEGRAM_WEB_HOOK); botToken = config.get(TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN); @@ -132,14 +134,13 @@ public final class TelegramSender { return result; } - private String sendInvoke(String title, String content) throws IOException { HttpPost httpPost = buildHttpPost(url, buildMsgJsonStr(content)); CloseableHttpClient httpClient; if (Boolean.TRUE.equals(enableProxy)) { if (StringUtils.isNotEmpty(user) && StringUtils.isNotEmpty(password)) { httpClient = getProxyClient(proxy, port, user, password); - }else { + } else { httpClient = getDefaultClient(); } RequestConfig rcf = getProxyConfig(proxy, port); @@ -180,6 +181,7 @@ public final class TelegramSender { } static class TelegramSendMsgResponse { + @JsonProperty("ok") private Boolean ok; @JsonProperty("error_code") diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml index 5668088eb7..fc1c373bee 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-webexteams jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java index f1b908ff6c..4a417909e4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.webexteams; public class WebexMessage { + private String roomId; private String toPersonId; private String toPersonEmail; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java index 08963cab6a..1a799ae50c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class WebexTeamsAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java index f17c3cde8f..e11078a31c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java @@ -32,6 +32,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class WebexTeamsAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "WebexTeams"; @@ -40,48 +41,63 @@ public final class WebexTeamsAlertChannelFactory implements AlertChannelFactory @Override public List params() { List paramsList = new ArrayList<>(); - InputParam botAccessToken = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_BOT_ACCESS_TOKEN, WebexTeamsParamsConstants.WEBEX_TEAMS_BOT_ACCESS_TOKEN) - .setPlaceholder("Please enter the robot's access token you were given") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam botAccessToken = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_BOT_ACCESS_TOKEN, + WebexTeamsParamsConstants.WEBEX_TEAMS_BOT_ACCESS_TOKEN) + .setPlaceholder("Please enter the robot's access token you were given") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam roomId = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_ROOM_ID, WebexTeamsParamsConstants.WEBEX_TEAMS_ROOM_ID) - .setPlaceholder("The room ID of the message") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam roomId = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_ROOM_ID, + WebexTeamsParamsConstants.WEBEX_TEAMS_ROOM_ID) + .setPlaceholder("The room ID of the message") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam toPersonId = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_ID, WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_ID) - .setPlaceholder("The person ID of the message recipient") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam toPersonId = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_ID, + WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_ID) + .setPlaceholder("The person ID of the message recipient") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam toPersonEmail = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_EMAIL, WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_EMAIL) - .setPlaceholder("The email address of the message recipient") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam toPersonEmail = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_EMAIL, + WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_EMAIL) + .setPlaceholder("The email address of the message recipient") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam atSomeoneInRoom = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_AT_SOMEONE_IN_ROOM, WebexTeamsParamsConstants.WEBEX_TEAMS_AT_SOMEONE_IN_ROOM) - .setPlaceholder("use ,(eng commas) to separate multiple emails") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam atSomeoneInRoom = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_AT_SOMEONE_IN_ROOM, + WebexTeamsParamsConstants.WEBEX_TEAMS_AT_SOMEONE_IN_ROOM) + .setPlaceholder("use ,(eng commas) to separate multiple emails") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - RadioParam destination = RadioParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_DESTINATION, WebexTeamsParamsConstants.WEBEX_TEAMS_DESTINATION) - .addParamsOptions(new ParamsOptions(WebexTeamsDestination.ROOM_ID.getDescp(), WebexTeamsDestination.ROOM_ID.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_EMAIL.getDescp(), WebexTeamsDestination.PERSON_EMAIL.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_ID.getDescp(), WebexTeamsDestination.PERSON_ID.getDescp(), false)) - .setValue(WebexTeamsDestination.ROOM_ID.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + RadioParam destination = RadioParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_DESTINATION, + WebexTeamsParamsConstants.WEBEX_TEAMS_DESTINATION) + .addParamsOptions(new ParamsOptions(WebexTeamsDestination.ROOM_ID.getDescp(), + WebexTeamsDestination.ROOM_ID.getDescp(), false)) + .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_EMAIL.getDescp(), + WebexTeamsDestination.PERSON_EMAIL.getDescp(), false)) + .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_ID.getDescp(), + WebexTeamsDestination.PERSON_ID.getDescp(), false)) + .setValue(WebexTeamsDestination.ROOM_ID.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); paramsList.add(botAccessToken); paramsList.add(roomId); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java index 1d23aed591..694fd3fdea 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java @@ -25,6 +25,7 @@ import java.util.Map; import com.google.common.base.Functions; public enum WebexTeamsDestination { + /** * 0 ROOM_ID; * 1 PERSON_EMAIL; @@ -51,7 +52,8 @@ public enum WebexTeamsDestination { } private static final Map WEBEX_TEAMS_DESTINATION_MAP = - Arrays.stream(WebexTeamsDestination.values()).collect(toMap(WebexTeamsDestination::getDescp, Functions.identity())); + Arrays.stream(WebexTeamsDestination.values()) + .collect(toMap(WebexTeamsDestination::getDescp, Functions.identity())); public static WebexTeamsDestination of(String descp) { if (WEBEX_TEAMS_DESTINATION_MAP.containsKey(descp)) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java index 7b61696c8f..adbbbe9eb1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import com.google.common.base.Preconditions; public final class WebexTeamsSender { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(WebexTeamsSender.class); private final String botAccessToken; @@ -57,7 +58,9 @@ public final class WebexTeamsSender { destination = WebexTeamsDestination.of(config.get(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_DESTINATION)); Preconditions.checkArgument(!Objects.isNull(botAccessToken), "WebexTeams bot access token can not be null"); Preconditions.checkArgument(!Objects.isNull(destination), "WebexTeams message destination can not be null"); - Preconditions.checkArgument((!Objects.isNull(roomId) || !Objects.isNull(toPersonId) || !Objects.isNull(toPersonEmail)),"WebexTeams message destination could not be determined. Provide only one destination in the roomId, toPersonEmail, or toPersonId field"); + Preconditions.checkArgument( + (!Objects.isNull(roomId) || !Objects.isNull(toPersonId) || !Objects.isNull(toPersonEmail)), + "WebexTeams message destination could not be determined. Provide only one destination in the roomId, toPersonEmail, or toPersonId field"); } public AlertResult sendWebexTeamsAlter(AlertData alertData) { @@ -137,11 +140,11 @@ public final class WebexTeamsSender { public static String addAtPersonEmailInRoom(String formatContent, String atPersonEmailInRoom) { String[] emailArr = atPersonEmailInRoom.split(","); StringBuilder formatContentBuilder = new StringBuilder(formatContent); - for (String email: emailArr) { + for (String email : emailArr) { formatContentBuilder.append(" <@personEmail:").append(email).append(">"); } - return formatContentBuilder.toString(); + return formatContentBuilder.toString(); } public static String formatContent(AlertData alertData) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml index b293ac6c1e..3576f0c5e2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-wechat jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java index 94b43ea300..786cdb159f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.alert.api.AlertResult; import java.util.Map; public final class WeChatAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo info) { AlertData alertData = info.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java index 39a4a8159f..a357756090 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java @@ -34,6 +34,7 @@ import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class WeChatAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "WeChat"; @@ -41,47 +42,58 @@ public final class WeChatAlertChannelFactory implements AlertChannelFactory { @Override public List params() { - InputParam corpIdParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_CORP_ID, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_CORP_ID) - .setPlaceholder("please input corp id ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam corpIdParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_CORP_ID, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_CORP_ID) + .setPlaceholder("please input corp id ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam secretParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SECRET, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SECRET) - .setPlaceholder("please input secret ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam secretParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SECRET, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SECRET) + .setPlaceholder("please input secret ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam usersParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_USERS, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_USERS) - .setPlaceholder("use `|` to separate userIds and `@all` to everyone ") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam usersParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_USERS, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_USERS) + .setPlaceholder("use `|` to separate userIds and `@all` to everyone ") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam agentIdParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_AGENT_ID, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_AGENT_ID) - .setPlaceholder("please input agent id or chat id ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam agentIdParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_AGENT_ID, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_AGENT_ID) + .setPlaceholder("please input agent id or chat id ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - RadioParam sendType = RadioParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SEND_TYPE, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SEND_TYPE) + RadioParam sendType = RadioParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SEND_TYPE, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SEND_TYPE) .addParamsOptions(new ParamsOptions(WeChatType.APP.getDescp(), WeChatType.APP.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WeChatType.APPCHAT.getDescp(), WeChatType.APPCHAT.getDescp(), false)) + .addParamsOptions( + new ParamsOptions(WeChatType.APPCHAT.getDescp(), WeChatType.APPCHAT.getDescp(), false)) .setValue(WeChatType.APP.getDescp()) .addValidate(Validate.newBuilder().setRequired(true).build()) .build(); RadioParam showType = RadioParam.newBuilder(AlertConstants.NAME_SHOW_TYPE, AlertConstants.SHOW_TYPE) - .addParamsOptions(new ParamsOptions(ShowType.MARKDOWN.getDescp(), ShowType.MARKDOWN.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) - .setValue(ShowType.MARKDOWN.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + .addParamsOptions(new ParamsOptions(ShowType.MARKDOWN.getDescp(), ShowType.MARKDOWN.getDescp(), false)) + .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) + .setValue(ShowType.MARKDOWN.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); return Arrays.asList(corpIdParam, secretParam, usersParam, agentIdParam, sendType, showType); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java index 36ebc1c8d4..7f5eaef4f9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.wechat; public final class WeChatAlertConstants { + static final String MARKDOWN_QUOTE = ">"; static final String MARKDOWN_ENTER = "\n"; @@ -29,7 +30,8 @@ public final class WeChatAlertConstants { static final String WE_CHAT_APP_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token" + "={token}"; - static final String WE_CHAT_TOKEN_URL = "https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}"; + static final String WE_CHAT_TOKEN_URL = + "https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}"; static final String WE_CHAT_CONTENT_KEY = "content"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java index d8085b901f..9511b04f88 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.wechat; public final class WeChatAlertParamsConstants { + static final String ENTERPRISE_WE_CHAT_CORP_ID = "$t('corpId')"; static final String NAME_ENTERPRISE_WE_CHAT_CORP_ID = "corpId"; static final String ENTERPRISE_WE_CHAT_SECRET = "$t('secret')"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index b8ed73d587..78408c5ba7 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -17,10 +17,16 @@ package org.apache.dolphinscheduler.plugin.alert.wechat; +import static java.util.Objects.requireNonNull; +import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.WE_CHAT_DUPLICATE_CHECK_INTERVAL_ZERO; +import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.WE_CHAT_ENABLE_ID_TRANS; +import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.WE_CHAT_MESSAGE_SAFE_PUBLICITY; + import org.apache.dolphinscheduler.alert.api.AlertConstants; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; + import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -29,8 +35,6 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; @@ -40,11 +44,11 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; - -import static java.util.Objects.requireNonNull; -import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class WeChatSender { + private static final Logger logger = LoggerFactory.getLogger(WeChatSender.class); private static final String MUST_NOT_NULL = " must not null"; private static final String ALERT_STATUS = "false"; @@ -210,11 +214,15 @@ public final class WeChatSender { String msgJson = ""; if (sendType.equals(WeChatType.APP.getDescp())) { enterpriseWeChatPushUrlReplace = WeChatAlertConstants.WE_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken); - WechatAppMessage wechatAppMessage = new WechatAppMessage(weChatUsers, showType, Integer.valueOf(weChatAgentIdChatId), contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY, WE_CHAT_ENABLE_ID_TRANS, WE_CHAT_DUPLICATE_CHECK_INTERVAL_ZERO); + WechatAppMessage wechatAppMessage = new WechatAppMessage(weChatUsers, showType, + Integer.valueOf(weChatAgentIdChatId), contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY, + WE_CHAT_ENABLE_ID_TRANS, WE_CHAT_DUPLICATE_CHECK_INTERVAL_ZERO); msgJson = JSONUtils.toJsonString(wechatAppMessage); } else if (sendType.equals(WeChatType.APPCHAT.getDescp())) { - enterpriseWeChatPushUrlReplace = WeChatAlertConstants.WE_CHAT_APP_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken); - WechatAppChatMessage wechatAppChatMessage = new WechatAppChatMessage(weChatAgentIdChatId, showType, contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY); + enterpriseWeChatPushUrlReplace = + WeChatAlertConstants.WE_CHAT_APP_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken); + WechatAppChatMessage wechatAppChatMessage = + new WechatAppChatMessage(weChatAgentIdChatId, showType, contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY); msgJson = JSONUtils.toJsonString(wechatAppChatMessage); } @@ -248,6 +256,7 @@ public final class WeChatSender { } static final class WeChatSendMsgResponse { + private Integer errcode; private String errmsg; @@ -302,7 +311,8 @@ public final class WeChatSender { } public String toString() { - return "WeChatSender.WeChatSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + this.getErrmsg() + ")"; + return "WeChatSender.WeChatSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + this.getErrmsg() + + ")"; } } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java index 9060b53089..109da59944 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java @@ -15,10 +15,10 @@ * limitations under the License. */ - package org.apache.dolphinscheduler.plugin.alert.wechat; public enum WeChatType { + APP(1, "APP/应用"), APPCHAT(2, "GROUP CHAT/群聊"), ; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java index e04f7335b8..9c16f759fa 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java @@ -25,8 +25,8 @@ public class WechatAppChatMessage { private String chatid; private String msgtype; - private Map text; - private Map markdown; + private Map text; + private Map markdown; private Integer safe; public String getChatid() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java index dbb74a6a93..f5ec7183ba 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java @@ -99,7 +99,8 @@ public class WechatAppMessage { public WechatAppMessage() { } - public WechatAppMessage(String touser, String msgtype, Integer agentid, Map contentMap, Integer safe, Integer enableIdTrans, Integer enableDuplicateCheck) { + public WechatAppMessage(String touser, String msgtype, Integer agentid, Map contentMap, + Integer safe, Integer enableIdTrans, Integer enableDuplicateCheck) { this.touser = touser; this.msgtype = msgtype; this.agentid = agentid; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml index a014f8b17c..df705e9ea2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert org.apache.dolphinscheduler + dolphinscheduler-alert dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-plugins pom diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java index 782bf0aa85..78ff13d987 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java @@ -30,11 +30,6 @@ import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.radio.RadioParam; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.HashMap; @@ -42,10 +37,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static java.lang.String.format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; @Component public final class AlertPluginManager { + private static final Logger logger = LoggerFactory.getLogger(AlertPluginManager.class); private final PluginDao pluginDao; @@ -59,20 +59,22 @@ public final class AlertPluginManager { private final PluginParams warningTypeParams = getWarningTypeParams(); public PluginParams getWarningTypeParams() { - return - RadioParam.newBuilder(AlertConstants.NAME_WARNING_TYPE, AlertConstants.WARNING_TYPE) - .addParamsOptions(new ParamsOptions(WarningType.SUCCESS.getDescp(), WarningType.SUCCESS.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WarningType.FAILURE.getDescp(), WarningType.FAILURE.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WarningType.ALL.getDescp(), WarningType.ALL.getDescp(), false)) - .setValue(WarningType.ALL.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + return RadioParam.newBuilder(AlertConstants.NAME_WARNING_TYPE, AlertConstants.WARNING_TYPE) + .addParamsOptions( + new ParamsOptions(WarningType.SUCCESS.getDescp(), WarningType.SUCCESS.getDescp(), false)) + .addParamsOptions( + new ParamsOptions(WarningType.FAILURE.getDescp(), WarningType.FAILURE.getDescp(), false)) + .addParamsOptions(new ParamsOptions(WarningType.ALL.getDescp(), WarningType.ALL.getDescp(), false)) + .setValue(WarningType.ALL.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); } @EventListener public void installPlugin(ApplicationReadyEvent readyEvent) { - PrioritySPIFactory prioritySPIFactory = new PrioritySPIFactory<>(AlertChannelFactory.class); + PrioritySPIFactory prioritySPIFactory = + new PrioritySPIFactory<>(AlertChannelFactory.class); for (Map.Entry entry : prioritySPIFactory.getSPIMap().entrySet()) { String name = entry.getKey(); AlertChannelFactory factory = entry.getValue(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java index 6bb4aa0049..930d9180be 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java @@ -34,6 +34,7 @@ import io.netty.channel.Channel; @Component public final class AlertRequestProcessor implements NettyRequestProcessor { + private static final Logger logger = LoggerFactory.getLogger(AlertRequestProcessor.class); private final AlertSenderService alertSenderService; @@ -44,18 +45,19 @@ public final class AlertRequestProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - checkArgument(CommandType.ALERT_SEND_REQUEST == command.getType(), "invalid command type: %s", command.getType()); + checkArgument(CommandType.ALERT_SEND_REQUEST == command.getType(), "invalid command type: %s", + command.getType()); AlertSendRequestCommand alertSendRequestCommand = JsonSerializer.deserialize( - command.getBody(), AlertSendRequestCommand.class); + command.getBody(), AlertSendRequestCommand.class); logger.info("Received command : {}", alertSendRequestCommand); AlertSendResponseCommand alertSendResponseCommand = alertSenderService.syncHandler( - alertSendRequestCommand.getGroupId(), - alertSendRequestCommand.getTitle(), - alertSendRequestCommand.getContent(), - alertSendRequestCommand.getWarnType()); + alertSendRequestCommand.getGroupId(), + alertSendRequestCommand.getTitle(), + alertSendRequestCommand.getContent(), + alertSendRequestCommand.getWarnType()); channel.writeAndFlush(alertSendResponseCommand.convert2Command(command.getOpaque())); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java index 0218e51d2b..7a8b89a52d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java @@ -19,10 +19,10 @@ package org.apache.dolphinscheduler.alert; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; @UtilityClass public class AlertServerMetrics { diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index ee7e379536..bcf840410d 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -17,18 +17,16 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 - pom dolphinscheduler-alert + pom dolphinscheduler-alert-api diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 08a73e69e3..2994208ea2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.enums.PluginType; import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginException; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; import org.apache.dolphinscheduler.spi.params.base.PluginParams; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java index db62bcda3a..dd62af88de 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java @@ -27,6 +27,7 @@ import java.lang.annotation.Target; @Retention(RetentionPolicy.RUNTIME) @Documented public @interface AccessLogAnnotation { + // ignore request args String[] ignoreRequestArgs() default {"loginUser"}; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java index b20b6d8425..6d78ada123 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java @@ -47,6 +47,7 @@ import org.springframework.web.context.request.ServletRequestAttributes; @Aspect @Component public class AccessLogAspect { + private static final Logger logger = LoggerFactory.getLogger(AccessLogAspect.class); private static final String TRACE_ID = "traceId"; @@ -56,7 +57,7 @@ public class AccessLogAspect { private static final Pattern sensitiveDataPattern = Pattern.compile(sensitiveDataRegEx, Pattern.CASE_INSENSITIVE); @Pointcut("@annotation(org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation)") - public void logPointCut(){ + public void logPointCut() { // Do nothing because of it's a pointcut } @@ -65,7 +66,7 @@ public class AccessLogAspect { long startTime = System.currentTimeMillis(); // fetch AccessLogAnnotation - MethodSignature sign = (MethodSignature) proceedingJoinPoint.getSignature(); + MethodSignature sign = (MethodSignature) proceedingJoinPoint.getSignature(); Method method = sign.getMethod(); AccessLogAnnotation annotation = method.getAnnotation(AccessLogAnnotation.class); @@ -73,7 +74,8 @@ public class AccessLogAspect { // log request if (!annotation.ignoreRequest()) { - ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + ServletRequestAttributes attributes = + (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); if (attributes != null) { HttpServletRequest request = attributes.getRequest(); String traceIdFromHeader = request.getHeader(TRACE_ID); @@ -92,7 +94,8 @@ public class AccessLogAspect { userName, request.getRequestURI(), request.getMethod(), - proceedingJoinPoint.getSignature().getDeclaringTypeName() + "." + proceedingJoinPoint.getSignature().getName(), + proceedingJoinPoint.getSignature().getDeclaringTypeName() + "." + + proceedingJoinPoint.getSignature().getName(), argsString); } @@ -102,7 +105,8 @@ public class AccessLogAspect { // log response if (!annotation.ignoreResponse()) { - logger.info("RESPONSE TRACE_ID:{}, BODY:{}, REQUEST DURATION:{} milliseconds", traceId, ob, (System.currentTimeMillis() - startTime)); + logger.info("RESPONSE TRACE_ID:{}, BODY:{}, REQUEST DURATION:{} milliseconds", traceId, ob, + (System.currentTimeMillis() - startTime)); } return ob; @@ -134,7 +138,7 @@ public class AccessLogAspect { boolean exists = false; while (matcher.find()) { if (matcher.groupCount() == 3) { - stream = IntStream.concat(stream, IntStream.range(matcher.end(1),matcher.end(2))); + stream = IntStream.concat(stream, IntStream.range(matcher.end(1), matcher.end(2))); exists = true; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java index 01ef5a28e8..0d936f0ad5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.dao.entity.User; import java.util.Date; public class AuditMessage { + private User user; private Date auditDate; @@ -34,7 +35,8 @@ public class AuditMessage { private Integer resourceId; - public AuditMessage(User user, Date auditDate, AuditResourceType resourceType, AuditOperationType operation, Integer resourceId) { + public AuditMessage(User user, Date auditDate, AuditResourceType resourceType, AuditOperationType operation, + Integer resourceId) { this.user = user; this.auditDate = auditDate; this.resourceType = resourceType; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java index 090b46888f..451fcbc40b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java @@ -25,6 +25,7 @@ import org.springframework.stereotype.Component; @EnableConfigurationProperties @ConfigurationProperties(value = "audit", ignoreUnknownFields = false) public class AuditConfiguration { + private boolean enabled; public boolean getEnabled() { @@ -34,4 +35,4 @@ public class AuditConfiguration { public void setEnabled(boolean enabled) { this.enabled = enabled; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java index 735f71f9ef..4e1c67bd0b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java @@ -18,20 +18,24 @@ package org.apache.dolphinscheduler.api.configuration; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; + +import springfox.documentation.service.Operation; +import springfox.documentation.service.RequestParameter; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spi.service.OperationBuilderPlugin; +import springfox.documentation.spi.service.contexts.OperationContext; + import java.util.Collection; import java.util.Locale; import java.util.Set; + +import lombok.RequiredArgsConstructor; + import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import lombok.RequiredArgsConstructor; -import springfox.documentation.service.Operation; -import springfox.documentation.service.RequestParameter; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spi.service.OperationBuilderPlugin; -import springfox.documentation.spi.service.contexts.OperationContext; @Component public class OpenAPITranslationConfiguration { @@ -60,20 +64,20 @@ public class OpenAPITranslationConfiguration { .collect(toSet()); Collection parameters = operation.getRequestParameters().stream() - .map(it -> new RequestParameter( - it.getName(), - it.getIn(), - messageSource.getMessage(it.getDescription(), null, it.getDescription(), locale), - it.getRequired(), - it.getDeprecated(), - it.getHidden(), - it.getParameterSpecification(), - it.getScalarExample(), - it.getExamples(), - it.getPrecedence(), - it.getExtensions(), - it.getParameterIndex())) - .collect(toList()); + .map(it -> new RequestParameter( + it.getName(), + it.getIn(), + messageSource.getMessage(it.getDescription(), null, it.getDescription(), locale), + it.getRequired(), + it.getDeprecated(), + it.getHidden(), + it.getParameterSpecification(), + it.getScalarExample(), + it.getExamples(), + it.getPrecedence(), + it.getExtensions(), + it.getParameterIndex())) + .collect(toList()); context.operationBuilder() .notes(notes) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java index 5735e27fd2..fcfa973424 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java @@ -25,6 +25,7 @@ import org.springframework.stereotype.Component; @EnableConfigurationProperties @ConfigurationProperties(value = "python-gateway", ignoreUnknownFields = false) public class PythonGatewayConfiguration { + private boolean enabled; private String gatewayServerAddress; private int gatewayServerPort; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java index 64b6cc3e43..649e32c088 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java @@ -24,6 +24,9 @@ import org.apache.dolphinscheduler.common.config.YamlPropertySourceFactory; import java.util.ArrayList; import java.util.List; +import lombok.Getter; +import lombok.Setter; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -31,9 +34,6 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; -import lombok.Getter; -import lombok.Setter; - @Component @EnableConfigurationProperties @PropertySource(value = {"classpath:task-type-config.yaml"}, factory = YamlPropertySourceFactory.class) @@ -62,9 +62,11 @@ public class TaskTypeConfiguration { universal.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_UNIVERSAL))); cloud.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_CLOUD))); logic.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_LOGIC))); - dataIntegration.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_DATA_INTEGRATION))); + dataIntegration + .forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_DATA_INTEGRATION))); dataQuality.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_DATA_QUALITY))); - machineLearning.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_MACHINE_LEARNING))); + machineLearning + .forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_MACHINE_LEARNING))); other.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_OTHER))); return defaultTaskTypes; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java index 3b66a2d6ca..47cb5230f3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.api.configuration; -import lombok.Data; - import java.util.HashMap; import java.util.Map; +import lombok.Data; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java index 035d582e06..b93792307f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.constants; import org.apache.dolphinscheduler.api.enums.ExecuteType; + import java.util.HashMap; import java.util.Map; @@ -30,7 +31,7 @@ public class ApiFuncIdentificationConstant { public static final String ACCESS_TOKEN_DELETE = "security:token:delete"; public static final String ALERT_GROUP_VIEW = "security:alert-group:view"; public static final String ALERT_GROUP_CREATE = "security:alert-group:create"; - public static final String ALERT_GROUP_UPDATE = "security:alert-group:update"; + public static final String ALERT_GROUP_UPDATE = "security:alert-group:update"; public static final String ALERT_GROUP_DELETE = "security:alert-group:delete"; public static final String TENANT_MANAGER = "security:tenant:view"; public static final String TENANT_CREATE = "security:tenant:create"; @@ -156,28 +157,28 @@ public class ApiFuncIdentificationConstant { public static final String MONITOR_STATISTICS_VIEW = "monitor:statistics:view"; - public final static Map map = new HashMap(); + public final static Map map = new HashMap(); - static{ - for(ExecuteType type : ExecuteType.values()){ - switch (type){ + static { + for (ExecuteType type : ExecuteType.values()) { + switch (type) { case REPEAT_RUNNING: - map.put(type,RERUN); + map.put(type, RERUN); break; case RECOVER_SUSPENDED_PROCESS: - map.put(type,RECOVERY_SUSPEND); + map.put(type, RECOVERY_SUSPEND); break; case START_FAILURE_TASK_PROCESS: - map.put(type,FAILED_TO_RETRY); + map.put(type, FAILED_TO_RETRY); break; case STOP: - map.put(type,STOP); + map.put(type, STOP); break; case PAUSE: - map.put(type,PAUSE); + map.put(type, PAUSE); break; case NONE: - map.put(type,null); + map.put(type, null); break; default: } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 82142768ce..8201eedfe2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AccessTokenService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index 6453de0276..853336b3b7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -30,8 +30,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AlertGroupService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java index a2ac7e0410..af805aa7e9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java @@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + import springfox.documentation.annotations.ApiIgnore; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java index c8dcdff039..e247688ec7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ClusterService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java index 04203651fb..9b4e371240 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java @@ -28,8 +28,8 @@ import org.apache.dolphinscheduler.api.service.DqExecuteResultService; import org.apache.dolphinscheduler.api.service.DqRuleService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index b344073664..3d9961e99b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -36,10 +36,10 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.service.utils.CommonUtils; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java index f42df4f4e1..e97ffcfe7b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java @@ -21,6 +21,7 @@ package org.apache.dolphinscheduler.api.controller; import javax.servlet.RequestDispatcher; import javax.servlet.http.HttpServletRequest; + import org.springframework.boot.web.servlet.error.ErrorController; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; @@ -29,6 +30,7 @@ import org.springframework.web.servlet.ModelAndView; @Controller public class DsErrorController implements ErrorController { + @RequestMapping("/error") public ModelAndView handleError(HttpServletRequest request) { Object status = request.getAttribute(RequestDispatcher.ERROR_STATUS_CODE); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java index 72ea72d1b0..fb1bf11380 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.EnvironmentService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 98e69ab1f4..f672702b85 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -121,7 +121,7 @@ public class ExecutorController extends BaseController { @ApiImplicitParam(name = "workerGroup", value = "WORKER_GROUP", dataType = "String", example = "default"), @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long", example = "-1"), @ApiImplicitParam(name = "timeout", value = "TIMEOUT", dataType = "Int", example = "100"), - @ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int" , example = "8"), + @ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int", example = "8"), @ApiImplicitParam(name = "dryRun", value = "DRY_RUN", dataType = "Int", example = "0"), @ApiImplicitParam(name = "testFlag", value = "TEST_FLAG", dataType = "Int", example = "0"), @ApiImplicitParam(name = "complementDependentMode", value = "COMPLEMENT_DEPENDENT_MODE", dataType = "complementDependentMode") @@ -166,7 +166,8 @@ public class ExecutorController extends BaseController { Map result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, - workerGroup, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, testFlag, complementDependentMode); + workerGroup, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, testFlag, + complementDependentMode); return returnDataList(result); } @@ -268,10 +269,12 @@ public class ExecutorController extends BaseController { complementDependentMode); if (!Status.SUCCESS.equals(result.get(Constants.STATUS))) { - logger.error("Process definition start failed, projectCode:{}, processDefinitionCode:{}.", projectCode, processDefinitionCode); + logger.error("Process definition start failed, projectCode:{}, processDefinitionCode:{}.", projectCode, + processDefinitionCode); startFailedProcessDefinitionCodeList.add(String.valueOf(processDefinitionCode)); } else - logger.info("Start process definition complete, projectCode:{}, processDefinitionCode:{}.", projectCode, processDefinitionCode); + logger.info("Start process definition complete, projectCode:{}, processDefinitionCode:{}.", projectCode, + processDefinitionCode); } if (!startFailedProcessDefinitionCodeList.isEmpty()) { @@ -303,9 +306,9 @@ public class ExecutorController extends BaseController { public Result execute(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectCode", value = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("processInstanceId") Integer processInstanceId, - @RequestParam("executeType") ExecuteType executeType - ) { - logger.info("Start to execute process instance, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + @RequestParam("executeType") ExecuteType executeType) { + logger.info("Start to execute process instance, projectCode:{}, processInstanceId:{}.", projectCode, + processInstanceId); Map result = execService.execute(loginUser, projectCode, processInstanceId, executeType); return returnDataList(result); } @@ -344,10 +347,12 @@ public class ExecutorController extends BaseController { Map singleResult = execService.execute(loginUser, projectCode, processInstanceId, executeType); if (!Status.SUCCESS.equals(singleResult.get(Constants.STATUS))) { - logger.error("Start to execute process instance error, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.error("Start to execute process instance error, projectCode:{}, processInstanceId:{}.", + projectCode, processInstanceId); executeFailedIdList.add((String) singleResult.get(Constants.MSG)); } else - logger.info("Start to execute process instance complete, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.info("Start to execute process instance complete, projectCode:{}, processInstanceId:{}.", + projectCode, processInstanceId); } catch (Exception e) { executeFailedIdList .add(MessageFormat.format(Status.PROCESS_INSTANCE_ERROR.getMsg(), strProcessInstanceId)); @@ -440,7 +445,8 @@ public class ExecutorController extends BaseController { startParamMap = JSONUtils.toMap(startParams); } - logger.info("Start to execute stream task instance, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}.", projectCode, code, version); + logger.info("Start to execute stream task instance, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}.", + projectCode, code, version); Map result = execService.execStreamTaskInstance(loginUser, projectCode, code, version, warningGroupId, workerGroup, environmentCode, startParamMap, dryRun); return returnDataList(result); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java index 98d38373f3..fb54b67283 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java @@ -17,8 +17,10 @@ package org.apache.dolphinscheduler.api.controller; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; +import static org.apache.dolphinscheduler.api.enums.Status.ADD_TASK_TYPE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_TASK_TYPE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR; + import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; import org.apache.dolphinscheduler.api.dto.FavTaskDto; import org.apache.dolphinscheduler.api.enums.Status; @@ -27,6 +29,13 @@ import org.apache.dolphinscheduler.api.service.FavTaskService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; + +import springfox.documentation.annotations.ApiIgnore; + +import java.util.List; + +import javax.annotation.Resource; + import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -36,14 +45,9 @@ import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import springfox.documentation.annotations.ApiIgnore; - -import javax.annotation.Resource; -import java.util.List; -import static org.apache.dolphinscheduler.api.enums.Status.ADD_TASK_TYPE_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.DELETE_TASK_TYPE_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; /** * fav controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java index 06a5adfdeb..9e95fd0e09 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java @@ -31,9 +31,9 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.K8sNamespaceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.K8sNamespace; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java index 9240ac45d8..1acbc99a40 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java @@ -28,6 +28,8 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; +import springfox.documentation.annotations.ApiIgnore; + import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; @@ -40,7 +42,6 @@ import org.springframework.web.bind.annotation.RestController; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; -import springfox.documentation.annotations.ApiIgnore; /** * monitor controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index 55c66a6adb..dc29875ec4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -38,26 +38,19 @@ import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_PROCESS_DEFINI import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; -import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum; import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; -import org.apache.commons.lang3.StringUtils; - -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import javax.servlet.http.HttpServletResponse; @@ -685,7 +678,8 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectCode", value = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("codes") String codes) { - Map result = processDefinitionService.batchDeleteProcessDefinitionByCodes(loginUser, projectCode, codes); + Map result = + processDefinitionService.batchDeleteProcessDefinitionByCodes(loginUser, projectCode, codes); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index b36ba8abd5..ebc5edde85 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -17,8 +17,17 @@ package org.apache.dolphinscheduler.api.controller; -import io.swagger.annotations.*; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.api.enums.Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_BY_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROCESS_INSTANCE_ERROR; + import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; @@ -26,14 +35,10 @@ import org.apache.dolphinscheduler.api.service.ProcessInstanceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + import springfox.documentation.annotations.ApiIgnore; import org.apache.commons.lang3.StringUtils; @@ -45,7 +50,26 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; /** * process instance controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index 76997f75dc..14c2cde3ee 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -32,8 +32,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java index 1dd6324a49..b167f9b47e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java @@ -42,8 +42,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java index 32734e811e..d71760be11 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java @@ -27,8 +27,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java index d00f63cb20..ec87292fba 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java @@ -36,8 +36,10 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + +import springfox.documentation.annotations.ApiIgnore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -55,7 +57,6 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiOperation; -import springfox.documentation.annotations.ApiIgnore; /** * queue controller @@ -93,9 +94,9 @@ public class QueueV2Controller extends BaseController { */ @ApiOperation(value = "queryQueueListPaging", notes = "QUERY_QUEUE_LIST_PAGING_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataTypeClass = String.class), - @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataTypeClass = int.class, example = "1"), - @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataTypeClass = int.class, example = "20") + @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataTypeClass = String.class), + @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataTypeClass = int.class, example = "1"), + @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataTypeClass = int.class, example = "20") }) @GetMapping() @ResponseStatus(HttpStatus.OK) @@ -109,7 +110,8 @@ public class QueueV2Controller extends BaseController { } String searchVal = ParameterUtils.handleEscapes(queueQueryRequest.getSearchVal()); - result = queueService.queryList(loginUser, searchVal, queueQueryRequest.getPageNo(), queueQueryRequest.getPageSize()); + result = queueService.queryList(loginUser, searchVal, queueQueryRequest.getPageNo(), + queueQueryRequest.getPageSize()); return new QueueListPagingResponse(result); } @@ -127,7 +129,8 @@ public class QueueV2Controller extends BaseController { @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public QueueCreateResponse createQueue(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestBody QueueCreateRequest queueCreateRequest) { - Result result = queueService.createQueue(loginUser, queueCreateRequest.getQueue(), queueCreateRequest.getQueueName()); + Result result = + queueService.createQueue(loginUser, queueCreateRequest.getQueue(), queueCreateRequest.getQueueName()); return new QueueCreateResponse(result); } @@ -141,16 +144,17 @@ public class QueueV2Controller extends BaseController { */ @ApiOperation(value = "updateQueue", notes = "UPDATE_QUEUE_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "id", value = "QUEUE_ID", required = true, dataTypeClass = int.class, example = "100") + @ApiImplicitParam(name = "id", value = "QUEUE_ID", required = true, dataTypeClass = int.class, example = "100") }) @PutMapping(value = "/{id}", consumes = {"application/json"}) @ResponseStatus(HttpStatus.CREATED) @ApiException(UPDATE_QUEUE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public QueueUpdateResponse updateQueue(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int id, @RequestBody QueueUpdateRequest queueUpdateRequest) { + @PathVariable(value = "id") int id, + @RequestBody QueueUpdateRequest queueUpdateRequest) { Result result = queueService.updateQueue(loginUser, id, queueUpdateRequest.getQueue(), - queueUpdateRequest.getQueueName()); + queueUpdateRequest.getQueueName()); return new QueueUpdateResponse(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index ea2c59c418..55ca04a1a0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -49,8 +49,8 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index 2f87a1bf00..45a815262d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + import springfox.documentation.annotations.ApiIgnore; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java index 301fe085fb..cf96586084 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java @@ -36,8 +36,8 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 9b7b300d4f..78ab34762d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -28,9 +28,9 @@ import org.apache.dolphinscheduler.api.service.TaskInstanceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index 8c99eb8cfd..c0724b6d34 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 1e427b0e23..44a03483d5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -39,8 +39,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.UsersService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java index f887f064b7..b22eda9eae 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java @@ -28,9 +28,9 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java index 7314a38eed..58681aed85 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java @@ -27,8 +27,8 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.WorkerGroupService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; @@ -73,11 +73,11 @@ public class WorkerGroupController extends BaseController { */ @ApiOperation(value = "saveWorkerGroup", notes = "CREATE_WORKER_GROUP_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "id", value = "WORKER_GROUP_ID", dataType = "Int", example = "10", defaultValue = "0"), - @ApiImplicitParam(name = "name", value = "WORKER_GROUP_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "addrList", value = "WORKER_ADDR_LIST", required = true, dataType = "String"), - @ApiImplicitParam(name = "description", value = "WORKER_DESC", required = false, dataType = "String"), - @ApiImplicitParam(name = "otherParamsJson", value = "WORKER_PARMS_JSON", required = false, dataType = "String"), + @ApiImplicitParam(name = "id", value = "WORKER_GROUP_ID", dataType = "Int", example = "10", defaultValue = "0"), + @ApiImplicitParam(name = "name", value = "WORKER_GROUP_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "addrList", value = "WORKER_ADDR_LIST", required = true, dataType = "String"), + @ApiImplicitParam(name = "description", value = "WORKER_DESC", required = false, dataType = "String"), + @ApiImplicitParam(name = "otherParamsJson", value = "WORKER_PARMS_JSON", required = false, dataType = "String"), }) @PostMapping() @ResponseStatus(HttpStatus.OK) @@ -87,10 +87,10 @@ public class WorkerGroupController extends BaseController { @RequestParam(value = "id", required = false, defaultValue = "0") int id, @RequestParam(value = "name") String name, @RequestParam(value = "addrList") String addrList, - @RequestParam(value = "description",required = false, defaultValue = "") String description, - @RequestParam(value = "otherParamsJson",required = false, defaultValue = "") String otherParamsJson - ) { - Map result = workerGroupService.saveWorkerGroup(loginUser, id, name, addrList, description, otherParamsJson); + @RequestParam(value = "description", required = false, defaultValue = "") String description, + @RequestParam(value = "otherParamsJson", required = false, defaultValue = "") String otherParamsJson) { + Map result = + workerGroupService.saveWorkerGroup(loginUser, id, name, addrList, description, otherParamsJson); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java index 5e6a518173..b1b0eb66c9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java @@ -28,7 +28,8 @@ public class CommandStateCount { private int normalCount; private CommandType commandState; - public CommandStateCount(){} + public CommandStateCount() { + } public CommandStateCount(int errorCount, int normalCount, CommandType commandState) { this.errorCount = errorCount; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java index 03190f4b9f..c0b8520e2e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.api.dto; - import io.swagger.annotations.ApiModelProperty; public class CreateTokenRequest { + @ApiModelProperty(example = "1", required = true) Integer userId; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java index e96e2b2690..d7fa93c40c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.entity.AccessToken; public class CreateTokenResponse extends Result { + private AccessToken data; public CreateTokenResponse(Result result) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java index d702e88b48..2ef0f11cbd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java @@ -40,7 +40,7 @@ public class RuleDefinition { public RuleDefinition() { } - public RuleDefinition(List ruleInputEntryList,List executeSqlList) { + public RuleDefinition(List ruleInputEntryList, List executeSqlList) { this.ruleInputEntryList = ruleInputEntryList; this.executeSqlList = executeSqlList; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index 66a42857a4..9e456109be 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.api.dto; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -28,10 +25,15 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @NoArgsConstructor @AllArgsConstructor public class TaskCountDto { + /** * total count */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java index 95dc18a49f..eb935659c9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.api.dto; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @Data @NoArgsConstructor diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java index c32454a784..401e3f6319 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java @@ -44,14 +44,13 @@ public class GanttDto { /** * task status map */ - private Map taskStatus; + private Map taskStatus; - - public GanttDto(){ + public GanttDto() { this.taskStatus = new HashMap<>(); - taskStatus.put("success","success"); + taskStatus.put("success", "success"); } - public GanttDto(int height, List tasks, List taskNames){ + public GanttDto(int height, List tasks, List taskNames) { this(); this.height = height; this.tasks = tasks; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java index 07243db3fd..5370f61e8a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java @@ -25,6 +25,7 @@ import java.util.List; * Task */ public class Task { + /** * task name */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java index aca6556db4..21ad4d75be 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.api.dto.project; -import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import io.swagger.annotations.ApiModelProperty; /** * project create request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java index a7ee4bd97b..b174adda84 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java @@ -35,4 +35,4 @@ public class ProjectDeleteResponse extends Result { this.setMsg(result.getMsg()); this.setData((Boolean) result.getData()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java index 2512650a73..942be6857d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java @@ -19,12 +19,13 @@ package org.apache.dolphinscheduler.api.dto.project; import org.apache.dolphinscheduler.api.dto.PageQueryDto; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * project query request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java index 32e36ac95a..69cb3b563c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.api.dto.project; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; - import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * project update request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java index 702826fe32..c318200f5a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.api.dto.queue; +import lombok.Data; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue create request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java index 727d8b12c6..3bdd2293cf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java @@ -19,12 +19,13 @@ package org.apache.dolphinscheduler.api.dto.queue; import org.apache.dolphinscheduler.api.dto.PageQueryDto; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue query request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java index fd04318a03..7d2bbdc42f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.api.dto.queue; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue update request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java index 88e5359843..6b07a7bb7d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.api.dto.queue; +import lombok.Data; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue verify request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java index 92aaf13320..bee5bc8590 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java @@ -19,6 +19,6 @@ package org.apache.dolphinscheduler.api.dto.resources; /** * file leaf */ -public class FileLeaf extends ResourceComponent{ +public class FileLeaf extends ResourceComponent { } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java index ce6ce3a011..2bcc2822a8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java @@ -24,5 +24,6 @@ import java.util.List; * interface filter */ public interface IFilter { + List filter(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java index bf5a49597c..6206769f7f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; * resource filter */ public class ResourceFilter implements IFilter { + /** * resource suffix */ @@ -40,7 +41,7 @@ public class ResourceFilter implements IFilter { /** * parent list */ - //Set parentList = new HashSet<>(); + // Set parentList = new HashSet<>(); /** * constructor @@ -56,7 +57,7 @@ public class ResourceFilter implements IFilter { * file filter * @return file filtered by suffix */ - public Set fileFilter(){ + public Set fileFilter() { return resourceList.stream().filter(t -> { String alias = t.getAlias(); return alias.endsWith(suffix); @@ -67,12 +68,12 @@ public class ResourceFilter implements IFilter { * list all parent dir * @return parent resource dir set */ - Set listAllParent(){ - Set parentList = new HashSet<>(); + Set listAllParent() { + Set parentList = new HashSet<>(); Set filterFileList = fileFilter(); - for(Resource file:filterFileList){ + for (Resource file : filterFileList) { parentList.add(file); - setAllParent(file,parentList); + setAllParent(file, parentList); } return parentList; @@ -83,11 +84,11 @@ public class ResourceFilter implements IFilter { * @param resource resource * @return parent resource dir set */ - private void setAllParent(Resource resource,Set parentList){ + private void setAllParent(Resource resource, Set parentList) { for (Resource resourceTemp : resourceList) { if (resourceTemp.getId() == resource.getPid()) { parentList.add(resourceTemp); - setAllParent(resourceTemp,parentList); + setAllParent(resourceTemp, parentList); } } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java index 391822e535..5b84fde1a5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.dto.resources.visitor; - import org.apache.dolphinscheduler.api.dto.resources.Directory; import org.apache.dolphinscheduler.api.dto.resources.FileLeaf; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; @@ -28,7 +27,7 @@ import java.util.List; /** * resource tree visitor */ -public class ResourceTreeVisitor implements Visitor{ +public class ResourceTreeVisitor implements Visitor { /** * resource list @@ -55,10 +54,10 @@ public class ResourceTreeVisitor implements Visitor{ ResourceComponent rootDirectory = new Directory(); for (Resource resource : resourceList) { // judge whether is root node - if (rootNode(resource)){ + if (rootNode(resource)) { ResourceComponent tempResourceComponent = getResourceComponent(resource); rootDirectory.add(tempResourceComponent); - tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(),resourceList)); + tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(), resourceList)); } } return rootDirectory; @@ -70,18 +69,18 @@ public class ResourceTreeVisitor implements Visitor{ * @param list resource list * @return resource component list */ - public static List setChildren(int id, List list ){ + public static List setChildren(int id, List list) { List childList = new ArrayList<>(); for (Resource resource : list) { - if (id == resource.getPid()){ + if (id == resource.getPid()) { ResourceComponent tempResourceComponent = getResourceComponent(resource); childList.add(tempResourceComponent); } } for (ResourceComponent resourceComponent : childList) { - resourceComponent.setChildren(setChildren(resourceComponent.getId(),list)); + resourceComponent.setChildren(setChildren(resourceComponent.getId(), list)); } - if (childList.size()==0){ + if (childList.size() == 0) { return new ArrayList<>(); } return childList; @@ -95,7 +94,7 @@ public class ResourceTreeVisitor implements Visitor{ public boolean rootNode(Resource resource) { boolean isRootNode = true; - if(resource.getPid() != -1 ){ + if (resource.getPid() != -1) { for (Resource parent : resourceList) { if (resource.getPid() == parent.getId()) { isRootNode = false; @@ -113,17 +112,17 @@ public class ResourceTreeVisitor implements Visitor{ */ private static ResourceComponent getResourceComponent(Resource resource) { ResourceComponent tempResourceComponent; - if(resource.isDirectory()){ + if (resource.isDirectory()) { tempResourceComponent = new Directory(); - }else{ + } else { tempResourceComponent = new FileLeaf(); } - + tempResourceComponent.setName(resource.getAlias()); - tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/","")); + tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/", "")); tempResourceComponent.setId(resource.getId()); tempResourceComponent.setPid(resource.getPid()); - tempResourceComponent.setIdValue(resource.getId(),resource.isDirectory()); + tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory()); tempResourceComponent.setDescription(resource.getDescription()); tempResourceComponent.setType(resource.getType()); return tempResourceComponent; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java index defbed13bb..382bcc5e16 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; * Visitor */ public interface Visitor { + /** * visit * @return resource component diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java index ff42d75845..f324c23c5a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java @@ -33,9 +33,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.swagger.annotations.ApiModelProperty; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - /** * schedule create request */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java index bfac404582..755daa1dd1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java @@ -33,7 +33,7 @@ public enum ExecuteType { NONE, REPEAT_RUNNING, RECOVER_SUSPENDED_PROCESS, START_FAILURE_TASK_PROCESS, STOP, PAUSE; public static ExecuteType getEnum(int value) { - for (ExecuteType e: ExecuteType.values()) { + for (ExecuteType e : ExecuteType.values()) { if (e.ordinal() == value) { return e; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java index 8cdf2b81f8..b4812a0307 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java @@ -31,5 +31,6 @@ import java.lang.annotation.Target; @Retention(RUNTIME) @Target(METHOD) public @interface ApiException { + Status value(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java index e1d5a47138..863ad79a06 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java @@ -55,4 +55,3 @@ public class ApiExceptionHandler { } } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java index 20041b64d4..19f121fff2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java @@ -56,13 +56,15 @@ public class RateLimitInterceptor implements HandlerInterceptor { .maximumSize(100) .expireAfterAccess(10, TimeUnit.MINUTES) .build(new CacheLoader() { + @Override public RateLimiter load(String token) { // use tenant customize rate limit Map customizeTenantQpsRate = trafficConfiguration.getCustomizeTenantQpsRate(); int tenantQuota = trafficConfiguration.getDefaultTenantQpsRate(); if (MapUtils.isNotEmpty(customizeTenantQpsRate)) { - tenantQuota = customizeTenantQpsRate.getOrDefault(token, trafficConfiguration.getDefaultTenantQpsRate()); + tenantQuota = customizeTenantQpsRate.getOrDefault(token, + trafficConfiguration.getDefaultTenantQpsRate()); } // use tenant default rate limit return RateLimiter.create(tenantQuota, 1, TimeUnit.SECONDS); @@ -70,7 +72,8 @@ public class RateLimitInterceptor implements HandlerInterceptor { }); @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException { + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, + Object handler) throws ExecutionException { // tenant-level rate limit if (trafficConfiguration.isTenantSwitch()) { String token = request.getHeader("token"); @@ -78,7 +81,8 @@ public class RateLimitInterceptor implements HandlerInterceptor { RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token); if (!tenantRateLimiter.tryAcquire()) { response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); - logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, tenantRateLimiter.getRate()); + logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, + tenantRateLimiter.getRate()); return false; } } @@ -87,7 +91,8 @@ public class RateLimitInterceptor implements HandlerInterceptor { if (trafficConfiguration.isGlobalSwitch()) { if (!globalRateLimiter.tryAcquire()) { response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); - logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate()); + logger.warn("Too many request, reach global rate limit, current qps is {}", + globalRateLimiter.getRate()); return false; } } @@ -97,7 +102,8 @@ public class RateLimitInterceptor implements HandlerInterceptor { public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) { this.trafficConfiguration = trafficConfiguration; if (trafficConfiguration.isGlobalSwitch()) { - this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); + this.globalRateLimiter = + RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java index dcfc6e734a..393f440ee4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java @@ -42,7 +42,8 @@ public class K8sClientService { @Autowired private K8sManager k8sManager; - public ResourceQuota upsertNamespaceAndResourceToK8s(K8sNamespace k8sNamespace, String yamlStr) throws RemotingException { + public ResourceQuota upsertNamespaceAndResourceToK8s(K8sNamespace k8sNamespace, + String yamlStr) throws RemotingException { upsertNamespaceToK8s(k8sNamespace.getNamespace(), k8sNamespace.getClusterCode()); return upsertNamespacedResourceToK8s(k8sNamespace, yamlStr); } @@ -61,48 +62,48 @@ public class K8sClientService { return getNamespaceFromK8s(name, clusterCode); } - private ResourceQuota upsertNamespacedResourceToK8s(K8sNamespace k8sNamespace, String yamlStr) throws RemotingException { + private ResourceQuota upsertNamespacedResourceToK8s(K8sNamespace k8sNamespace, + String yamlStr) throws RemotingException { KubernetesClient client = k8sManager.getK8sClient(k8sNamespace.getClusterCode()); - //创建资源 + // 创建资源 ResourceQuota queryExist = client.resourceQuotas() - .inNamespace(k8sNamespace.getNamespace()) - .withName(k8sNamespace.getNamespace()) - .get(); + .inNamespace(k8sNamespace.getNamespace()) + .withName(k8sNamespace.getNamespace()) + .get(); ResourceQuota body = yaml.loadAs(yamlStr, ResourceQuota.class); if (queryExist != null) { if (k8sNamespace.getLimitsCpu() == null && k8sNamespace.getLimitsMemory() == null) { client.resourceQuotas().inNamespace(k8sNamespace.getNamespace()) - .withName(k8sNamespace.getNamespace()) - .delete(); + .withName(k8sNamespace.getNamespace()) + .delete(); return null; } } return client.resourceQuotas().inNamespace(k8sNamespace.getNamespace()) - .withName(k8sNamespace.getNamespace()) - .createOrReplace(body); + .withName(k8sNamespace.getNamespace()) + .createOrReplace(body); } private Optional getNamespaceFromK8s(String name, Long clusterCode) throws RemotingException { NamespaceList listNamespace = - k8sManager.getK8sClient(clusterCode).namespaces().list(); + k8sManager.getK8sClient(clusterCode).namespaces().list(); Optional list = - listNamespace.getItems().stream() - .filter((Namespace namespace) -> - namespace.getMetadata().getName().equals(name)) - .findFirst(); + listNamespace.getItems().stream() + .filter((Namespace namespace) -> namespace.getMetadata().getName().equals(name)) + .findFirst(); return list; } private Namespace upsertNamespaceToK8s(String name, Long clusterCode) throws RemotingException { Optional result = getNamespaceFromK8s(name, clusterCode); - //if not exist create + // if not exist create if (!result.isPresent()) { KubernetesClient client = k8sManager.getK8sClient(clusterCode); Namespace body = new Namespace(); @@ -115,4 +116,4 @@ public class K8sClientService { return result.get(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java index fd9a249328..422075cbd7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java @@ -67,7 +67,8 @@ public class K8sManager { * @param clusterCode * @return new client if need updated */ - public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode, boolean update) throws RemotingException { + public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode, + boolean update) throws RemotingException { if (null == clusterCode) { return null; } @@ -84,7 +85,6 @@ public class K8sManager { return clientMap.get(clusterCode); } - private void deleteK8sClientInner(Long clusterCode) { if (clusterCode == null) { return; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java index 41d69e590e..3d2e42b81b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java @@ -31,6 +31,7 @@ import java.util.List; import org.slf4j.Logger; public class PermissionCheck { + /** * logger */ @@ -74,7 +75,8 @@ public class PermissionCheck { /** * permission check */ - public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, + int userId) { this.authorizationType = authorizationType; this.processService = processService; this.needChecks = needChecks; @@ -84,7 +86,8 @@ public class PermissionCheck { /** * permission check */ - public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, + int userId, Logger logger) { this.authorizationType = authorizationType; this.processService = processService; this.needChecks = needChecks; @@ -95,7 +98,8 @@ public class PermissionCheck { /** * permission check */ - public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List resourceList, int userId, Logger logger) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, + List resourceList, int userId, Logger logger) { this.authorizationType = authorizationType; this.processService = processService; this.resourceList = resourceList; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java index eea1f14f98..b8016c5926 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java @@ -16,13 +16,13 @@ */ package org.apache.dolphinscheduler.api.permission; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; -import org.slf4j.Logger; - import java.util.List; import java.util.Set; -public interface ResourcePermissionCheckService{ +import org.slf4j.Logger; + +public interface ResourcePermissionCheckService { + /** * resourcePermissionCheck * @param authorizationType @@ -51,7 +51,8 @@ public interface ResourcePermissionCheckService{ * @param logger * @return */ - boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, String permissionKey, Logger logger); + boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, + String permissionKey, Logger logger); /** * functionDisabled diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java index 1ebd187369..49c35242b2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java @@ -35,7 +35,6 @@ package org.apache.dolphinscheduler.api.permission; import static java.util.stream.Collectors.toSet; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.AccessToken; @@ -52,9 +51,7 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; -import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper; import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -67,6 +64,8 @@ import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.commons.collections.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -86,23 +85,29 @@ import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; @Component -public class ResourcePermissionCheckServiceImpl implements ResourcePermissionCheckService, ApplicationContextAware { +public class ResourcePermissionCheckServiceImpl + implements + ResourcePermissionCheckService, + ApplicationContextAware { @Autowired private ProcessService processService; - public static final Map> RESOURCE_LIST_MAP = new ConcurrentHashMap<>(); + public static final Map> RESOURCE_LIST_MAP = + new ConcurrentHashMap<>(); @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - for (ResourceAcquisitionAndPermissionCheck authorizedResourceList : applicationContext.getBeansOfType(ResourceAcquisitionAndPermissionCheck.class).values()) { + for (ResourceAcquisitionAndPermissionCheck authorizedResourceList : applicationContext + .getBeansOfType(ResourceAcquisitionAndPermissionCheck.class).values()) { List authorizationTypes = authorizedResourceList.authorizationTypes(); authorizationTypes.forEach(auth -> RESOURCE_LIST_MAP.put(auth, authorizedResourceList)); } - } + } @Override - public boolean resourcePermissionCheck(Object authorizationType, Object[] needChecks, Integer userId, Logger logger) { + public boolean resourcePermissionCheck(Object authorizationType, Object[] needChecks, Integer userId, + Logger logger) { if (Objects.nonNull(needChecks) && needChecks.length > 0) { Set originResSet = new HashSet<>(Arrays.asList(needChecks)); Set ownResSets = RESOURCE_LIST_MAP.get(authorizationType).listAuthorizedResource(userId, logger); @@ -115,7 +120,8 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe } @Override - public boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, String permissionKey, Logger logger) { + public boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, + String permissionKey, Logger logger) { User user = processService.getUserById(userId); if (user == null) { logger.error("User does not exist, userId:{}.", userId); @@ -229,7 +235,8 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe } else { // query resource relation List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 0); - relationResources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); + relationResources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() + : resourceMapper.queryResourceListById(resIds); } List ownResourceList = resourceMapper.queryResourceListAuthored(userId, -1); relationResources.addAll(ownResourceList); @@ -259,7 +266,7 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe @Override public Set listAuthorizedResource(int userId, Logger logger) { List udfFuncList = udfFuncMapper.listAuthorizedUdfByUserId(userId); - if (udfFuncList.isEmpty()){ + if (udfFuncList.isEmpty()) { return Collections.emptySet(); } return udfFuncList.stream().map(UdfFunc::getId).collect(toSet()); @@ -325,7 +332,6 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe } } - @Component public static class EnvironmentResourceList implements ResourceAcquisitionAndPermissionCheck { @@ -342,7 +348,7 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe @Override public boolean permissionCheck(int userId, String url, Logger logger) { - return true; + return true; } @Override @@ -374,7 +380,6 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { return Collections.emptySet(); @@ -403,7 +408,6 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { return Collections.emptySet(); @@ -429,10 +433,9 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe @Override public boolean permissionCheck(int userId, String url, Logger logger) { - return false; + return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { List alertGroupList = alertGroupMapper.queryAllGroupList(); @@ -462,7 +465,6 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { if (userId != 0) { @@ -495,10 +497,10 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe return true; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { - return dataSourceMapper.listAuthorizedDataSource(userId, null).stream().map(DataSource::getId).collect(toSet()); + return dataSourceMapper.listAuthorizedDataSource(userId, null).stream().map(DataSource::getId) + .collect(toSet()); } } @@ -526,11 +528,11 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe @Override public Set listAuthorizedResource(int userId, Logger logger) { - return accessTokenMapper.listAuthorizedAccessToken(userId, null).stream().map(AccessToken::getId).collect(toSet()); + return accessTokenMapper.listAuthorizedAccessToken(userId, null).stream().map(AccessToken::getId) + .collect(toSet()); } } - interface ResourceAcquisitionAndPermissionCheck { /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 89fcc5fcf3..c465c3f9fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -17,19 +17,6 @@ package org.apache.dolphinscheduler.api.python; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import javax.annotation.PostConstruct; - -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.api.configuration.PythonGatewayConfiguration; import org.apache.dolphinscheduler.api.dto.EnvironmentDto; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; @@ -45,7 +32,6 @@ import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.service.UsersService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; @@ -57,7 +43,6 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; @@ -75,14 +60,30 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import py4j.GatewayServer; + +import org.apache.commons.collections.CollectionUtils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import py4j.GatewayServer; @Component public class PythonGateway { + private static final Logger logger = LoggerFactory.getLogger(PythonGateway.class); private static final FailureStrategy DEFAULT_FAILURE_STRATEGY = FailureStrategy.CONTINUE; @@ -147,6 +148,7 @@ public class PythonGateway { // TODO replace this user to build in admin user if we make sure build in one could not be change private final User dummyAdminUser = new User() { + { setId(ADMIN_USER_ID); setUserName("dummyUser"); @@ -155,6 +157,7 @@ public class PythonGateway { }; private final Queue queuePythonGateway = new Queue() { + { setId(Integer.MAX_VALUE); setQueueName("queuePythonGateway"); @@ -171,7 +174,8 @@ public class PythonGateway { return taskDefinitionService.genTaskCodeList(genNum); } - public Map getCodeAndVersion(String projectName, String processDefinitionName, String taskName) throws CodeGenerateUtils.CodeGenerateException { + public Map getCodeAndVersion(String projectName, String processDefinitionName, + String taskName) throws CodeGenerateUtils.CodeGenerateException { Project project = projectMapper.queryByName(projectName); Map result = new HashMap<>(); // project do not exists, mean task not exists too, so we should directly return init value @@ -181,15 +185,18 @@ public class PythonGateway { return result; } - ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); - // In the case project exists, but current process definition still not created, we should also return the init version of it + ProcessDefinition processDefinition = + processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); + // In the case project exists, but current process definition still not created, we should also return the init + // version of it if (processDefinition == null) { result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("version", 0L); return result; } - TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); + TaskDefinition taskDefinition = + taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); if (taskDefinition == null) { result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("version", 0L); @@ -249,21 +256,26 @@ public class PythonGateway { if (processDefinition != null) { processDefinitionCode = processDefinition.getCode(); // make sure process definition offline which could edit - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.OFFLINE); - Map result = processDefinitionService.updateProcessDefinition(user, projectCode, name, processDefinitionCode, description, globalParams, + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.OFFLINE); + Map result = processDefinitionService.updateProcessDefinition(user, projectCode, name, + processDefinitionCode, description, globalParams, null, timeout, tenantCode, taskRelationJson, taskDefinitionJson, otherParamsJson, executionType); } else { - Map result = processDefinitionService.createProcessDefinition(user, projectCode, name, description, globalParams, - null, timeout, tenantCode, taskRelationJson, taskDefinitionJson, otherParamsJson, executionType); + Map result = processDefinitionService.createProcessDefinition(user, projectCode, name, + description, globalParams, + null, timeout, tenantCode, taskRelationJson, taskDefinitionJson, otherParamsJson, executionType); processDefinition = (ProcessDefinition) result.get(Constants.DATA_LIST); processDefinitionCode = processDefinition.getCode(); } // Fresh process definition schedule if (schedule != null) { - createOrUpdateSchedule(user, projectCode, processDefinitionCode, schedule, workerGroup, warningType, warningGroupId); + createOrUpdateSchedule(user, projectCode, processDefinitionCode, schedule, workerGroup, warningType, + warningGroupId); } - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.getEnum(releaseState)); + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.getEnum(releaseState)); return processDefinitionCode; } @@ -275,14 +287,16 @@ public class PythonGateway { * @param processDefinitionName process definition name */ private ProcessDefinition getProcessDefinition(User user, long projectCode, String processDefinitionName) { - Map verifyProcessDefinitionExists = processDefinitionService.verifyProcessDefinitionName(user, projectCode, processDefinitionName, 0); + Map verifyProcessDefinitionExists = + processDefinitionService.verifyProcessDefinitionName(user, projectCode, processDefinitionName, 0); Status verifyStatus = (Status) verifyProcessDefinitionExists.get(Constants.STATUS); ProcessDefinition processDefinition = null; if (verifyStatus == Status.PROCESS_DEFINITION_NAME_EXIST) { processDefinition = processDefinitionMapper.queryByDefineName(projectCode, processDefinitionName); } else if (verifyStatus != Status.SUCCESS) { - String msg = "Verify process definition exists status is invalid, neither SUCCESS or PROCESS_DEFINITION_NAME_EXIST."; + String msg = + "Verify process definition exists status is invalid, neither SUCCESS or PROCESS_DEFINITION_NAME_EXIST."; logger.error(msg); throw new RuntimeException(msg); } @@ -314,13 +328,16 @@ public class PythonGateway { // create or update schedule int scheduleId; if (scheduleObj == null) { - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.ONLINE); - Map result = schedulerService.insertSchedule(user, projectCode, processDefinitionCode, schedule, WarningType.valueOf(warningType), + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.ONLINE); + Map result = schedulerService.insertSchedule(user, projectCode, processDefinitionCode, + schedule, WarningType.valueOf(warningType), warningGroupId, DEFAULT_FAILURE_STRATEGY, DEFAULT_PRIORITY, workerGroup, DEFAULT_ENVIRONMENT_CODE); scheduleId = (int) result.get("scheduleId"); } else { scheduleId = scheduleObj.getId(); - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.OFFLINE); + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.OFFLINE); schedulerService.updateSchedule(user, projectCode, scheduleId, schedule, WarningType.valueOf(warningType), warningGroupId, DEFAULT_FAILURE_STRATEGY, DEFAULT_PRIORITY, workerGroup, DEFAULT_ENVIRONMENT_CODE); } @@ -334,14 +351,15 @@ public class PythonGateway { String workerGroup, String warningType, Integer warningGroupId, - Integer timeout - ) { + Integer timeout) { User user = usersService.queryUser(userName); Project project = projectMapper.queryByName(projectName); - ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); + ProcessDefinition processDefinition = + processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); // make sure process definition online - processDefinitionService.releaseProcessDefinition(user, project.getCode(), processDefinition.getCode(), ReleaseState.ONLINE); + processDefinitionService.releaseProcessDefinition(user, project.getCode(), processDefinition.getCode(), + ReleaseState.ONLINE); executorService.execProcessInstance(user, project.getCode(), @@ -362,14 +380,13 @@ public class PythonGateway { null, DEFAULT_DRY_RUN, DEFAULT_TEST_FLAG, - COMPLEMENT_DEPENDENT_MODE - ); + COMPLEMENT_DEPENDENT_MODE); } // side object /* - Grant project's permission to user. Use when project's created user not current but - Python API use it to change process definition. + * Grant project's permission to user. Use when project's created user not current but Python API use it to change + * process definition. */ private Integer grantProjectToUser(Project project, User user) { Date now = new Date(); @@ -383,8 +400,8 @@ public class PythonGateway { } /* - Grant or create project. Create a new project if project do not exists, and grant the project - permission to user if project exists but without permission to this user. + * Grant or create project. Create a new project if project do not exists, and grant the project permission to user + * if project exists but without permission to this user. */ public void createOrGrantProject(String userName, String name, String desc) { User user = usersService.queryUser(userName); @@ -452,7 +469,8 @@ public class PythonGateway { return user; } - public User updateUser(String userName, String userPassword, String email, String phone, String tenantCode, String queue, int state) throws Exception { + public User updateUser(String userName, String userPassword, String email, String phone, String tenantCode, + String queue, int state) throws Exception { return usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state); } @@ -496,7 +514,8 @@ public class PythonGateway { * @param projectName project name which process definition belongs to * @param processDefinitionName process definition name */ - public Map getProcessDefinitionInfo(String userName, String projectName, String processDefinitionName) { + public Map getProcessDefinitionInfo(String userName, String projectName, + String processDefinitionName) { Map result = new HashMap<>(); User user = usersService.queryUser(userName); @@ -506,7 +525,8 @@ public class PythonGateway { // get process definition info if (processDefinition != null) { // make sure process definition online - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinition.getCode(), ReleaseState.ONLINE); + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinition.getCode(), + ReleaseState.ONLINE); result.put("id", processDefinition.getId()); result.put("name", processDefinition.getName()); result.put("code", processDefinition.getCode()); @@ -539,7 +559,8 @@ public class PythonGateway { long projectCode = project.getCode(); result.put("projectCode", projectCode); - ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(projectCode, processDefinitionName); + ProcessDefinition processDefinition = + processDefinitionMapper.queryByDefineName(projectCode, processDefinitionName); if (processDefinition == null) { String msg = String.format("Can not find valid process definition by name %s", processDefinitionName); logger.error(msg); @@ -548,7 +569,8 @@ public class PythonGateway { result.put("processDefinitionCode", processDefinition.getCode()); if (taskName != null) { - TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(projectCode, processDefinition.getCode(), taskName); + TaskDefinition taskDefinition = + taskDefinitionMapper.queryByName(projectCode, processDefinition.getCode(), taskName); result.put("taskDefinitionCode", taskDefinition.getCode()); } return result; @@ -564,11 +586,14 @@ public class PythonGateway { public Map getResourcesFileInfo(String programType, String fullName) { Map result = new HashMap<>(); - Result resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE, ProgramType.valueOf(programType)); + Result resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE, + ProgramType.valueOf(programType)); List resourcesComponent = (List) resources.getData(); - List namedResources = resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList()); + List namedResources = + resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList()); if (CollectionUtils.isEmpty(namedResources)) { - String msg = String.format("Can not find valid resource by program type %s and name %s", programType, fullName); + String msg = + String.format("Can not find valid resource by program type %s and name %s", programType, fullName); logger.error(msg); throw new IllegalArgumentException(msg); } @@ -596,7 +621,6 @@ public class PythonGateway { return environmentDto.getCode(); } - /** * Get resource by given resource type and full name. It return map contain resource id, name. * Useful in Python API create task which need processDefinition information. @@ -623,7 +647,8 @@ public class PythonGateway { * @return id of resource */ public Integer createOrUpdateResource( - String userName, String fullName, String description, String resourceContent) { + String userName, String fullName, String description, + String resourceContent) { return resourceService.createOrUpdateResource(userName, fullName, description, resourceContent); } @@ -640,15 +665,14 @@ public class PythonGateway { InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfiguration.getGatewayServerAddress()); InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfiguration.getPythonAddress()); server = new GatewayServer( - this, - pythonGatewayConfiguration.getGatewayServerPort(), - pythonGatewayConfiguration.getPythonPort(), - gatewayHost, - pythonHost, - pythonGatewayConfiguration.getConnectTimeout(), - pythonGatewayConfiguration.getReadTimeout(), - null - ); + this, + pythonGatewayConfiguration.getGatewayServerPort(), + pythonGatewayConfiguration.getPythonPort(), + gatewayHost, + pythonHost, + pythonGatewayConfiguration.getConnectTimeout(), + pythonGatewayConfiguration.getReadTimeout(), + null); GatewayServer.turnLoggingOn(); logger.info("PythonGatewayService started on: " + gatewayHost.toString()); server.start(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java index 99769b840b..5c2a556bb7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.servlet.http.HttpServletRequest; public interface Authenticator { + /** * Verifying legality via username and password * @param username user name diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java index 6a17068e22..3e3bb1ae79 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java @@ -32,6 +32,7 @@ import org.springframework.context.annotation.Configuration; @Configuration public class SecurityConfig { + private static final Logger logger = LoggerFactory.getLogger(SecurityConfig.class); @Value("${security.authentication.type:PASSWORD}") diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java index 8512aaf248..24587948c1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java @@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; public abstract class AbstractAuthenticator implements Authenticator { + private static final Logger logger = LoggerFactory.getLogger(AbstractAuthenticator.class); @Autowired @@ -106,7 +107,7 @@ public abstract class AbstractAuthenticator implements Authenticator { logger.info("session info is null "); return null; } - //get user object from session + // get user object from session return userService.queryUser(session.getUserId()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java index 394ed44065..e849f884a5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java @@ -32,7 +32,7 @@ public class LdapAuthenticator extends AbstractAuthenticator { User user = null; String ldapEmail = ldapService.ldapLogin(userId, password); if (ldapEmail != null) { - //check if user exist + // check if user exist user = userService.getUserByUserName(userId); if (user == null && ldapService.createIfUserNotExists()) { user = userService.createUser(ldapService.getUserType(userId), userId, ldapEmail); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 1668cffe37..2dbfc4dce5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -57,7 +57,6 @@ public interface AccessTokenService { */ Result createToken(User loginUser, int userId, String expireTime, String token); - /** * generate token * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 8d7e49af51..538518b5ff 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -76,7 +76,8 @@ public interface AlertGroupService { * @param alertInstanceIds alertInstanceIds * @return update result code */ - Map updateAlertgroup(User loginUser, int id, String groupName, String desc, String alertInstanceIds); + Map updateAlertgroup(User loginUser, int id, String groupName, String desc, + String alertInstanceIds); /** * delete alert group by id diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java index 5296f31e11..24c7ce5100 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java @@ -36,7 +36,7 @@ public interface AlertPluginInstanceService { * @param pluginInstanceParams plugin instance params * @return result */ - Map create(User loginUser,int pluginDefineId,String instanceName,String pluginInstanceParams); + Map create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams); /** * update @@ -46,7 +46,8 @@ public interface AlertPluginInstanceService { * @param pluginInstanceParams plugin instance params * @return result */ - Map update(User loginUser, int alertPluginInstanceId,String instanceName,String pluginInstanceParams); + Map update(User loginUser, int alertPluginInstanceId, String instanceName, + String pluginInstanceParams); /** * delete alert plugin instance diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java index a1fa21a9f1..6c86aeb0b2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.api.service; -import java.util.List; -import java.util.Map; - import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.List; +import java.util.Map; + import org.slf4j.Logger; /** @@ -39,7 +40,6 @@ public interface BaseService { */ boolean isAdmin(User user); - /** * isNotAdmin * @@ -86,7 +86,6 @@ public interface BaseService { */ boolean check(Map result, boolean bool, Status userNoOperationPerm); - /** * Verify that the operator has permissions * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java index 0787e5db41..0b542579db 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java @@ -96,4 +96,3 @@ public interface ClusterService { Map verifyCluster(String clusterName); } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index cb4639acfc..389731682e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.User; + import org.apache.ibatis.annotations.Param; import java.util.Date; @@ -50,7 +51,8 @@ public interface DataAnalysisService { * @param endDate end date * @return process instance state count data */ - Map countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate); + Map countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, + String endDate); /** * statistics the process definition quantities of a certain person diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index eaf1552caa..c1454f948b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -142,5 +142,5 @@ public interface DataSourceService { * @param tableName * @return */ - Map getTableColumns(Integer datasourceId,String tableName); + Map getTableColumns(Integer datasourceId, String tableName); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java index d0350556ac..df6d68d71e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java @@ -38,5 +38,5 @@ public interface DqRuleService { String endTime, Integer pageNo, Integer pageSize); - Map getDatasourceOptionsById(int datasourceId); + Map getDatasourceOptionsById(int datasourceId); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java index 617d2b26ac..ebedea43fe 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java @@ -52,7 +52,6 @@ public interface EnvironmentService { */ Map queryEnvironmentByCode(Long code); - /** * delete environment * @@ -71,7 +70,8 @@ public interface EnvironmentService { * @param desc environment desc * @param workerGroups worker groups */ - Map updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, String workerGroups); + Map updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, + String workerGroups); /** * query environment paging @@ -100,4 +100,3 @@ public interface EnvironmentService { Map verifyEnvironment(String environmentName); } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java index 9db770158d..4d22814e5f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java @@ -38,4 +38,3 @@ public interface EnvironmentWorkerGroupRelationService { */ Map queryAllEnvironmentWorkerGroupRelationList(); } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 9aaabb36b6..6f19a29caf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -61,7 +61,8 @@ public interface ExecutorService { Map execProcessInstance(User loginUser, long projectCode, long processDefinitionCode, String cronTime, CommandType commandType, FailureStrategy failureStrategy, String startNodeList, - TaskDependType taskDependType, WarningType warningType, Integer warningGroupId, + TaskDependType taskDependType, WarningType warningType, + Integer warningGroupId, RunMode runMode, Priority processInstancePriority, String workerGroup, Long environmentCode, Integer timeout, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java index d3f15a4dd6..7a14169acc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java @@ -50,7 +50,8 @@ public interface K8sNamespaceService { * @param limitsMemory limits memory, can null means not limit * @return */ - Map createK8sNamespace(User loginUser, String namespace, Long clusterCode, Double limitsCpu, Integer limitsMemory); + Map createK8sNamespace(User loginUser, String namespace, Long clusterCode, Double limitsCpu, + Integer limitsMemory); /** * update K8s Namespace tag and resource limit @@ -61,7 +62,8 @@ public interface K8sNamespaceService { * @param limitsMemory max memory * @return */ - Map updateK8sNamespace(User loginUser, int id, String userName, Double limitsCpu, Integer limitsMemory); + Map updateK8sNamespace(User loginUser, int id, String userName, Double limitsCpu, + Integer limitsMemory); /** * verify namespace and k8s @@ -106,4 +108,4 @@ public interface K8sNamespaceService { * @return namespace list */ List queryNamespaceAvailable(User loginUser); -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index f7ba946e4c..33477c509e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -38,7 +38,6 @@ public interface LoggerService { */ Result queryLog(int taskInstId, int skipLineNum, int limit); - /** * get log size * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index d301957955..e881b2e346 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -34,23 +34,23 @@ public interface MonitorService { * @param loginUser login user * @return data base state */ - Map queryDatabaseState(User loginUser); - + Map queryDatabaseState(User loginUser); + /** * query master list * * @param loginUser login user * @return master information list */ - Map queryMaster(User loginUser); - + Map queryMaster(User loginUser); + /** * query worker list * * @param loginUser login user * @return worker information list */ - Map queryWorker(User loginUser); - + Map queryWorker(User loginUser); + List getServerListFromRegistry(boolean isMaster); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index e887d0f017..8019cc40ae 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -245,8 +245,8 @@ public interface ProcessDefinitionService { * @return delete result code */ Map batchDeleteProcessDefinitionByCodes(User loginUser, - long projectCode, - String codes); + long projectCode, + String codes); /** * delete process definition by code diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index ca03353740..d84ca5d3cb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -18,16 +18,16 @@ package org.apache.dolphinscheduler.api.service; -import java.io.IOException; -import java.util.List; -import java.util.Map; - import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; +import java.io.IOException; +import java.util.List; +import java.util.Map; + /** * process instance service */ @@ -201,4 +201,4 @@ public interface ProcessInstanceService { */ List queryByProcessDefineCode(Long processDefinitionCode, int size); -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index 553cde34b5..0f62ea9028 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.User; -import java.util.Map; - /** * queue service */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 00c6fa151b..4de830c074 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -98,7 +98,8 @@ public interface ResourcesService { * @param pageSize page size * @return resource list page */ - Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize); + Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo, + Integer pageSize); /** * query resource list @@ -135,7 +136,7 @@ public interface ResourcesService { * @param type resource type * @return true if the resource name not exists, otherwise return false */ - Result verifyResourceName(String fullName, ResourceType type,User loginUser); + Result verifyResourceName(String fullName, ResourceType type, User loginUser); /** * verify resource by full name or pid and type @@ -144,7 +145,7 @@ public interface ResourcesService { * @param type resource type * @return true if the resource full name or pid not exists, otherwise return false */ - Result queryResource(User loginUser,String fullName,Integer id,ResourceType type); + Result queryResource(User loginUser, String fullName, Integer id, ResourceType type); /** * view resource file online @@ -154,7 +155,7 @@ public interface ResourcesService { * @param limit limit * @return resource content */ - Result readResource(User loginUser,int resourceId, int skipLineNum, int limit); + Result readResource(User loginUser, int resourceId, int skipLineNum, int limit); /** * create resource file online @@ -167,7 +168,8 @@ public interface ResourcesService { * @param content content * @return create result code */ - Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory); + Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, + String desc, String content, int pid, String currentDirectory); /** * create or update resource. @@ -179,7 +181,8 @@ public interface ResourcesService { * @param content content of resource * @return create result code */ - Result onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFullName, String desc, String content); + Result onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFullName, String desc, + String content); /** * create or update resource. @@ -200,7 +203,7 @@ public interface ResourcesService { * @param content content * @return update result cod */ - Result updateResourceContent(User loginUser,int resourceId, String content); + Result updateResourceContent(User loginUser, int resourceId, String content); /** * download file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java index e6e81d950b..a0d05c5b8d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java @@ -35,11 +35,11 @@ public interface TaskGroupQueueService { * @param status Task queue status * @param pageNo page no * @param pageSize page size - + * @return tasks list */ - Map queryTasksByGroupId(User loginUser, String taskName - , String processName, Integer status, int groupId, int pageNo,int pageSize); + Map queryTasksByGroupId(User loginUser, String taskName, String processName, Integer status, + int groupId, int pageNo, int pageSize); /** * query tasks in task group queue by project id @@ -68,7 +68,7 @@ public interface TaskGroupQueueService { */ boolean deleteByTaskId(int taskId); - void forceStartTask(int queueId,int forceStart); + void forceStartTask(int queueId, int forceStart); void modifyPriority(Integer queueId, Integer priority); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java index 4b15ca878f..3987109733 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java @@ -66,7 +66,7 @@ public interface TaskGroupService { * @param pageSize page size * @return the result code and msg */ - Map queryAllTaskGroup(User loginUser, String name,Integer status, int pageNo, int pageSize); + Map queryAllTaskGroup(User loginUser, String name, Integer status, int pageNo, int pageSize); /** * query all task group by status diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index 0d795a10c4..fc5abf4b6b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -66,7 +66,7 @@ public interface TenantService { * @throws Exception exception */ Map updateTenant(User loginUser, int id, String tenantCode, int queueId, - String desc) throws Exception; + String desc) throws Exception; /** * delete tenant diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 1f5d1ab339..0cac41663f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.dao.entity.User; -import java.util.Map; - /** * udf func service */ @@ -56,7 +54,7 @@ public interface UdfFuncService { * @param id udf function id * @return udf function detail */ - Result queryUdfFuncDetail(User loginUser,int id); + Result queryUdfFuncDetail(User loginUser, int id); /** * updateProcessInstance udf function @@ -72,14 +70,14 @@ public interface UdfFuncService { * @return update result code */ Result updateUdfFunc(User loginUser, - int udfFuncId, - String funcName, - String className, - String argTypes, - String database, - String desc, - UdfType type, - int resourceId); + int udfFuncId, + String funcName, + String className, + String argTypes, + String database, + String desc, + UdfType type, + int resourceId); /** * query udf function list paging @@ -117,4 +115,4 @@ public interface UdfFuncService { */ Result verifyUdfFuncByName(User loginUser, String name); -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index b4555ad505..d854e9afdd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -130,7 +130,8 @@ public interface UsersService { * @throws Exception exception */ Map updateUser(User loginUser, int userId, String userName, String userPassword, String email, - int tenantId, String phone, String queue, int state, String timeZone) throws IOException; + int tenantId, String phone, String queue, int state, + String timeZone) throws IOException; /** * delete user @@ -152,7 +153,6 @@ public interface UsersService { */ Map grantProject(User loginUser, int userId, String projectIds); - /** * grant project by code * @@ -182,7 +182,6 @@ public interface UsersService { */ Map grantResources(User loginUser, int userId, String resourceIds); - /** * grant udf function * @@ -193,7 +192,6 @@ public interface UsersService { */ Map grantUDFFunction(User loginUser, int userId, String udfIds); - /** * grant namespace * @@ -204,7 +202,6 @@ public interface UsersService { */ Map grantNamespaces(User loginUser, int userId, String namespaceIds); - /** * grant datasource * @@ -231,7 +228,6 @@ public interface UsersService { */ Map queryAllGeneralUsers(User loginUser); - /** * query user list * @@ -248,7 +244,6 @@ public interface UsersService { */ Result verifyUserName(String userName); - /** * unauthorized user * @@ -258,7 +253,6 @@ public interface UsersService { */ Map unauthorizedUser(User loginUser, Integer alertGroupId); - /** * authorized user * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index 4c474b75cd..b2f9d78178 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -38,7 +38,8 @@ public interface WorkerGroupService { * @param otherParamsJson otherParamsJson * @return create or update result code */ - Map saveWorkerGroup(User loginUser, int id, String name, String addrList, String description, String otherParamsJson); + Map saveWorkerGroup(User loginUser, int id, String name, String addrList, String description, + String otherParamsJson); /** * query worker group paging diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java index 42b56a0909..17e0729f9c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java @@ -125,7 +125,8 @@ public class AccessTokenServiceImpl extends BaseServiceImpl implements AccessTok Result result = new Result(); // 1. check permission - if (!(canOperatorPermissions(loginUser,null, AuthorizationType.ACCESS_TOKEN,ACCESS_TOKEN_CREATE) || loginUser.getId() == userId)) { + if (!(canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN, ACCESS_TOKEN_CREATE) + || loginUser.getId() == userId)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -190,7 +191,7 @@ public class AccessTokenServiceImpl extends BaseServiceImpl implements AccessTok @Override public Map delAccessTokenById(User loginUser, int id) { Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN,ACCESS_TOKEN_DELETE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN, ACCESS_TOKEN_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -226,7 +227,7 @@ public class AccessTokenServiceImpl extends BaseServiceImpl implements AccessTok Map result = new HashMap<>(); // 1. check permission - if (!canOperatorPermissions(loginUser, null,AuthorizationType.ACCESS_TOKEN,ACCESS_TOKEN_UPDATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN, ACCESS_TOKEN_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java index 2b528acf46..145c9aee84 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java @@ -17,6 +17,11 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_UPDATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_VIEW; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.AlertGroupService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -48,8 +53,6 @@ import org.springframework.transaction.annotation.Transactional; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; - /** * alert group service impl */ @@ -74,7 +77,8 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { alertGroups = alertGroupMapper.queryAllGroupList(); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); putMsg(result, Status.SUCCESS); @@ -99,12 +103,12 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup Map result = new HashMap<>(); result.put(Constants.STATUS, false); - //only admin can operate - if (!canOperatorPermissions(loginUser,new Object[]{id}, AuthorizationType.ALERT_GROUP,ALERT_GROUP_VIEW)) { + // only admin can operate + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.ALERT_GROUP, ALERT_GROUP_VIEW)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - //check if exist + // check if exist AlertGroup alertGroup = alertGroupMapper.selectById(id); if (alertGroup == null) { putMsg(result, Status.ALERT_GROUP_NOT_EXIST); @@ -134,7 +138,8 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { alertGroupPage = alertGroupMapper.queryAlertGroupPage(page, searchVal); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -161,14 +166,15 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup */ @Override @Transactional - public Map createAlertgroup(User loginUser, String groupName, String desc, String alertInstanceIds) { + public Map createAlertgroup(User loginUser, String groupName, String desc, + String alertInstanceIds) { Map result = new HashMap<>(); - //only admin can operate - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_GROUP, ALERT_GROUP_CREATE)) { + // only admin can operate + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_GROUP, ALERT_GROUP_CREATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -189,7 +195,8 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup if (insert > 0) { result.put(Constants.DATA_LIST, alertGroup); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.ALERT_GROUP, loginUser.getId(), Collections.singletonList(alertGroup.getId()), logger); + permissionPostHandle(AuthorizationType.ALERT_GROUP, loginUser.getId(), + Collections.singletonList(alertGroup.getId()), logger); logger.info("Create alert group complete, groupName:{}", alertGroup.getGroupName()); } else { logger.error("Create alert group error, groupName:{}", alertGroup.getGroupName()); @@ -214,14 +221,15 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup * @return update result code */ @Override - public Map updateAlertgroup(User loginUser, int id, String groupName, String desc, String alertInstanceIds) { + public Map updateAlertgroup(User loginUser, int id, String groupName, String desc, + String alertInstanceIds) { Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser, new Object[]{id},AuthorizationType.ALERT_GROUP,ALERT_GROUP_UPDATE)) { + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.ALERT_GROUP, ALERT_GROUP_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -268,8 +276,8 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup Map result = new HashMap<>(); result.put(Constants.STATUS, false); - //only admin can operate - if (!canOperatorPermissions(loginUser,new Object[]{id}, AuthorizationType.ALERT_GROUP,ALERT_GROUP_DELETE)) { + // only admin can operate + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.ALERT_GROUP, ALERT_GROUP_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -281,7 +289,7 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup return result; } - //check exist + // check exist AlertGroup alertGroup = alertGroupMapper.selectById(id); if (alertGroup == null) { logger.error("Alert group does not exist, id:{}.", id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java index ee95249209..fa7b44362a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java @@ -17,6 +17,10 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALART_INSTANCE_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_UPDATE; + import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService; @@ -55,8 +59,6 @@ import org.springframework.stereotype.Service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; - /** * alert plugin instance service impl */ @@ -84,7 +86,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A * @param pluginInstanceParams plugin instance params */ @Override - public Map create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams) { + public Map create(User loginUser, int pluginDefineId, String instanceName, + String pluginInstanceParams) { AlertPluginInstance alertPluginInstance = new AlertPluginInstance(); String paramsMapJson = parsePluginParamsMap(pluginInstanceParams); alertPluginInstance.setPluginInstanceParams(paramsMapJson); @@ -92,12 +95,13 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A alertPluginInstance.setPluginDefineId(pluginDefineId); Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ALART_INSTANCE_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALART_INSTANCE_CREATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } if (alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()) == Boolean.TRUE) { - logger.error("Plugin instance with the same name already exists, name:{}.", alertPluginInstance.getInstanceName()); + logger.error("Plugin instance with the same name already exists, name:{}.", + alertPluginInstance.getInstanceName()); putMsg(result, Status.PLUGIN_INSTANCE_ALREADY_EXIT); return result; } @@ -123,14 +127,16 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A * @param pluginInstanceParams plugin instance params */ @Override - public Map update(User loginUser, int pluginInstanceId, String instanceName, String pluginInstanceParams) { + public Map update(User loginUser, int pluginInstanceId, String instanceName, + String pluginInstanceParams) { String paramsMapJson = parsePluginParamsMap(pluginInstanceParams); - AlertPluginInstance alertPluginInstance = new AlertPluginInstance(pluginInstanceId, paramsMapJson, instanceName, new Date()); + AlertPluginInstance alertPluginInstance = + new AlertPluginInstance(pluginInstanceId, paramsMapJson, instanceName, new Date()); Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ALERT_PLUGIN_UPDATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALERT_PLUGIN_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -158,14 +164,14 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A @Override public Map delete(User loginUser, int id) { Map result = new HashMap<>(); - //check if there is an associated alert group + // check if there is an associated alert group boolean hasAssociatedAlertGroup = checkHasAssociatedAlertGroup(String.valueOf(id)); if (hasAssociatedAlertGroup) { logger.warn("Delete alert plugin failed because alert group is using it, pluginId:{}.", id); putMsg(result, Status.DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED); return result; } - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ALERT_PLUGIN_DELETE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALERT_PLUGIN_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -190,7 +196,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A public Map get(User loginUser, int id) { Map result = new HashMap<>(); AlertPluginInstance alertPluginInstance = alertPluginInstanceMapper.selectById(id); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ApiFuncIdentificationConstant.ALARM_INSTANCE_MANAGE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, + ApiFuncIdentificationConstant.ALARM_INSTANCE_MANAGE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -225,7 +232,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A Result result = new Result(); Page page = new Page<>(pageNo, pageSize); - IPage alertPluginInstanceIPage = alertPluginInstanceMapper.queryByInstanceNamePage(page, searchVal); + IPage alertPluginInstanceIPage = + alertPluginInstanceMapper.queryByInstanceNamePage(page, searchVal); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); pageInfo.setTotal((int) alertPluginInstanceIPage.getTotal()); @@ -243,7 +251,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A if (CollectionUtils.isEmpty(pluginDefineList)) { return null; } - Map pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity())); + Map pluginDefineMap = + pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity())); List alertPluginInstanceVOS = new ArrayList<>(); alertPluginInstances.forEach(alertPluginInstance -> { AlertPluginInstanceVO alertPluginInstanceVO = new AlertPluginInstanceVO(); @@ -254,12 +263,13 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A alertPluginInstanceVO.setInstanceName(alertPluginInstance.getInstanceName()); alertPluginInstanceVO.setId(alertPluginInstance.getId()); PluginDefine pluginDefine = pluginDefineMap.get(alertPluginInstance.getPluginDefineId()); - //FIXME When the user removes the plug-in, this will happen. At this time, maybe we should add a new field to indicate that the plug-in has expired? + // FIXME When the user removes the plug-in, this will happen. At this time, maybe we should add a new field + // to indicate that the plug-in has expired? if (null == pluginDefine) { return; } alertPluginInstanceVO.setAlertPluginName(pluginDefine.getPluginName()); - //todo List pages do not recommend returning this parameter + // todo List pages do not recommend returning this parameter String pluginParamsMapString = alertPluginInstance.getPluginInstanceParams(); String uiPluginParams = parseToPluginUiParams(pluginParamsMapString, pluginDefine.getPluginParams()); alertPluginInstanceVO.setPluginInstanceParams(uiPluginParams); @@ -288,7 +298,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A * @return Complete parameters list(include ui) */ private String parseToPluginUiParams(String pluginParamsMapString, String pluginUiParams) { - List> pluginParamsList = PluginParamsTransfer.generatePluginParams(pluginParamsMapString, pluginUiParams); + List> pluginParamsList = + PluginParamsTransfer.generatePluginParams(pluginParamsMapString, pluginUiParams); return JSONUtils.toJsonString(pluginParamsList); } @@ -297,7 +308,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A if (CollectionUtils.isEmpty(idsList)) { return false; } - Optional first = idsList.stream().filter(k -> null != k && Arrays.asList(k.split(",")).contains(id)).findFirst(); + Optional first = + idsList.stream().filter(k -> null != k && Arrays.asList(k.split(",")).contains(id)).findFirst(); return first.isPresent(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index 2ae6740838..bc798c9eea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -88,7 +88,7 @@ public class AuditServiceImpl extends BaseServiceImpl implements AuditService { Map checkAndParseDateResult = checkAndParseDateParameters(startDate, endDate); Status resultEnum = (Status) checkAndParseDateResult.get(Constants.STATUS); if (resultEnum != Status.SUCCESS) { - putMsg(result,resultEnum); + putMsg(result, resultEnum); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java index d8d24925b2..ad0a9fe828 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java @@ -17,15 +17,6 @@ package org.apache.dolphinscheduler.api.service.impl; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.service.BaseService; @@ -35,6 +26,16 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.User; + +import org.apache.commons.lang3.StringUtils; + +import java.text.MessageFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -43,16 +44,18 @@ import org.springframework.beans.factory.annotation.Autowired; * base service impl */ public class BaseServiceImpl implements BaseService { + private static final Logger logger = LoggerFactory.getLogger(BaseServiceImpl.class); @Autowired protected ResourcePermissionCheckService resourcePermissionCheckService; @Override - public void permissionPostHandle(AuthorizationType authorizationType, Integer userId, List ids, Logger logger) { - try{ + public void permissionPostHandle(AuthorizationType authorizationType, Integer userId, List ids, + Logger logger) { + try { resourcePermissionCheckService.postHandle(authorizationType, userId, ids, logger); - }catch (Exception e){ + } catch (Exception e) { logger.error("Post handle error, userId:{}.", userId, e); throw new RuntimeException("Resource association user error", e); } @@ -78,7 +81,7 @@ public class BaseServiceImpl implements BaseService { */ @Override public boolean isNotAdmin(User loginUser, Map result) { - //only admin can operate + // only admin can operate if (!isAdmin(loginUser)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return true; @@ -145,14 +148,14 @@ public class BaseServiceImpl implements BaseService { * @param tenantCode tenant code * @throws IOException if hdfs operation exception */ -// @Override -// public void createTenantDirIfNotExists(String tenantCode) throws IOException { -// String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); -// String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); -// // init resource path and udf path -// HadoopUtils.getInstance().mkdir(tenantCode,resourcePath); -// HadoopUtils.getInstance().mkdir(tenantCode,udfsPath); -// } + // @Override + // public void createTenantDirIfNotExists(String tenantCode) throws IOException { + // String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); + // String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); + // // init resource path and udf path + // HadoopUtils.getInstance().mkdir(tenantCode,resourcePath); + // HadoopUtils.getInstance().mkdir(tenantCode,udfsPath); + // } /** * Verify that the operator has permissions @@ -173,9 +176,11 @@ public class BaseServiceImpl implements BaseService { * @return boolean */ @Override - public boolean canOperatorPermissions(User user, Object[] ids,AuthorizationType type,String permissionKey) { - boolean operationPermissionCheck = resourcePermissionCheckService.operationPermissionCheck(type, type.equals(AuthorizationType.PROJECTS) ? ids : null, user.getId(), permissionKey, logger); - boolean resourcePermissionCheck = resourcePermissionCheckService.resourcePermissionCheck(type, ids, user.getUserType().equals(UserType.ADMIN_USER) ? 0 : user.getId(), logger); + public boolean canOperatorPermissions(User user, Object[] ids, AuthorizationType type, String permissionKey) { + boolean operationPermissionCheck = resourcePermissionCheckService.operationPermissionCheck(type, + type.equals(AuthorizationType.PROJECTS) ? ids : null, user.getId(), permissionKey, logger); + boolean resourcePermissionCheck = resourcePermissionCheckService.resourcePermissionCheck(type, ids, + user.getUserType().equals(UserType.ADMIN_USER) ? 0 : user.getId(), logger); return operationPermissionCheck && resourcePermissionCheck; } @@ -217,6 +222,6 @@ public class BaseServiceImpl implements BaseService { @Override public boolean checkDescriptionLength(String description) { - return description!=null && description.codePointCount(0, description.length()) > 255; + return description != null && description.codePointCount(0, description.length()) > 255; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index b7d4e01d5e..cdd40cbc0f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -258,7 +258,8 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic .selectCount(new QueryWrapper().lambda().eq(K8sNamespace::getClusterCode, code)); if (relatedNamespaceNumber > 0) { - logger.warn("Delete cluster failed because {} namespace(s) is(are) using it, clusterCode:{}.", relatedNamespaceNumber, code); + logger.warn("Delete cluster failed because {} namespace(s) is(are) using it, clusterCode:{}.", + relatedNamespaceNumber, code); putMsg(result, Status.DELETE_CLUSTER_RELATED_NAMESPACE_EXISTS); return result; } @@ -274,7 +275,6 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic return result; } - /** * update cluster * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java index 276e1ddb42..da6b5c156e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.api.service.impl; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.PROJECT_OVERVIEW; + import org.apache.dolphinscheduler.api.dto.CommandStateCount; import org.apache.dolphinscheduler.api.dto.DefineUserDto; import org.apache.dolphinscheduler.api.dto.TaskCountDto; @@ -44,10 +44,9 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import java.util.ArrayList; import java.util.Arrays; @@ -60,7 +59,10 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.PROJECT_OVERVIEW; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; /** * data analysis service impl diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 808f85311d..6547d83490 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -71,8 +71,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.fasterxml.jackson.databind.node.ObjectNode; - - /** * data source service impl */ @@ -87,7 +85,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource @Autowired private DataSourceUserMapper datasourceUserMapper; - private static final String TABLE = "TABLE"; private static final String VIEW = "VIEW"; private static final String[] TABLE_TYPES = new String[]{TABLE, VIEW}; @@ -106,7 +103,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource public Result createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam) { DataSourceUtils.checkDatasourceParam(datasourceParam); Result result = new Result<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.DATASOURCE, ApiFuncIdentificationConstant.DATASOURCE_CREATE_DATASOURCE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.DATASOURCE, + ApiFuncIdentificationConstant.DATASOURCE_CREATE_DATASOURCE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -116,7 +114,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource putMsg(result, Status.DATASOURCE_EXIST); return result; } - if(checkDescriptionLength(datasourceParam.getNote())){ + if (checkDescriptionLength(datasourceParam.getNote())) { logger.warn("Parameter description is too long, description:{}.", datasourceParam.getNote()); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -146,8 +144,10 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource try { dataSourceMapper.insert(dataSource); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.DATASOURCE, loginUser.getId(), Collections.singletonList(dataSource.getId()), logger); - logger.info("Datasource create complete, dbType:{}, datasourceName:{}.", dataSource.getType().getDescp(), dataSource.getName()); + permissionPostHandle(AuthorizationType.DATASOURCE, loginUser.getId(), + Collections.singletonList(dataSource.getId()), logger); + logger.info("Datasource create complete, dbType:{}, datasourceName:{}.", dataSource.getType().getDescp(), + dataSource.getName()); } catch (DuplicateKeyException ex) { logger.error("Datasource create error.", ex); putMsg(result, Status.DATASOURCE_EXIST); @@ -175,24 +175,26 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource return result; } - if (!canOperatorPermissions(loginUser,new Object[]{dataSource.getId()}, AuthorizationType.DATASOURCE, DATASOURCE_UPDATE)) { + if (!canOperatorPermissions(loginUser, new Object[]{dataSource.getId()}, AuthorizationType.DATASOURCE, + DATASOURCE_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - //check name can use or not + // check name can use or not if (!dataSource.getName().trim().equals(dataSource.getName()) && checkName(dataSource.getName())) { logger.warn("Datasource with the same name already exists, name:{}.", dataSource.getName()); putMsg(result, Status.DATASOURCE_EXIST); return result; } - if(checkDescriptionLength(dataSourceParam.getNote())){ + if (checkDescriptionLength(dataSourceParam.getNote())) { logger.warn("Parameter description is too long, description:{}.", dataSourceParam.getNote()); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; } - //check password，if the password is not updated, set to the old password. - BaseConnectionParam connectionParam = (BaseConnectionParam) DataSourceUtils.buildConnectionParams(dataSourceParam); + // check password，if the password is not updated, set to the old password. + BaseConnectionParam connectionParam = + (BaseConnectionParam) DataSourceUtils.buildConnectionParams(dataSourceParam); String password = connectionParam.getPassword(); if (StringUtils.isBlank(password)) { String oldConnectionParams = dataSource.getConnectionParams(); @@ -213,17 +215,19 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource dataSource.setType(dataSource.getType()); dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam)); dataSource.setUpdateTime(now); - if(dataSource.getTestFlag() == 1 && dataSourceParam.getTestFlag() == 0){ + if (dataSource.getTestFlag() == 1 && dataSourceParam.getTestFlag() == 0) { clearBindTestId(id); } dataSource.setTestFlag(dataSourceParam.getTestFlag()); dataSource.setBindTestId(dataSourceParam.getBindTestId()); try { dataSourceMapper.updateById(dataSource); - logger.info("Update datasource complete, datasourceId:{}, datasourceName:{}.", dataSource.getId(), dataSource.getName()); + logger.info("Update datasource complete, datasourceId:{}, datasourceName:{}.", dataSource.getId(), + dataSource.getName()); putMsg(result, Status.SUCCESS); } catch (DuplicateKeyException ex) { - logger.error("Update datasource error, datasourceId:{}, datasourceName:{}.", dataSource.getId(), dataSource.getName()); + logger.error("Update datasource error, datasourceId:{}, datasourceName:{}.", dataSource.getId(), + dataSource.getName()); putMsg(result, Status.DATASOURCE_EXIST); } return result; @@ -279,9 +283,11 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource Page dataSourcePage = new Page<>(pageNo, pageSize); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { - dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, UserType.ADMIN_USER.equals(loginUser.getUserType()) ? 0 : loginUser.getId(), searchVal); + dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, + UserType.ADMIN_USER.equals(loginUser.getUserType()) ? 0 : loginUser.getId(), searchVal); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -335,13 +341,16 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { datasourceList = dataSourceMapper.queryDataSourceByType(0, type, testFlag); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); putMsg(result, Status.SUCCESS); return result; } - datasourceList = dataSourceMapper.selectBatchIds(ids).stream().filter(dataSource -> dataSource.getType().getCode() == type).filter(dataSource -> dataSource.getTestFlag() == testFlag).collect(Collectors.toList()); + datasourceList = dataSourceMapper.selectBatchIds(ids).stream() + .filter(dataSource -> dataSource.getType().getCode() == type) + .filter(dataSource -> dataSource.getTestFlag() == testFlag).collect(Collectors.toList()); } result.put(Constants.DATA_LIST, datasourceList); @@ -360,7 +369,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource Result result = new Result<>(); List dataSourceList = dataSourceMapper.queryDataSourceByName(name); if (dataSourceList != null && !dataSourceList.isEmpty()) { - logger.warn("Datasource with the same name already exists, dataSourceName:{}.", dataSourceList.get(0).getName()); + logger.warn("Datasource with the same name already exists, dataSourceName:{}.", + dataSourceList.get(0).getName()); putMsg(result, Status.DATASOURCE_EXIST); } else { putMsg(result, Status.SUCCESS); @@ -382,18 +392,21 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource Result result = new Result<>(); try (Connection connection = DataSourceClientProvider.getInstance().getConnection(type, connectionParam)) { if (connection == null) { - logger.error("Connection test to {} datasource failed, connectionParam:{}.", type.getDescp(), connectionParam); + logger.error("Connection test to {} datasource failed, connectionParam:{}.", type.getDescp(), + connectionParam); putMsg(result, Status.CONNECTION_TEST_FAILURE); return result; } - logger.info("Connection test to {} datasource success, connectionParam:{}", type.getDescp(), connectionParam); + logger.info("Connection test to {} datasource success, connectionParam:{}", type.getDescp(), + connectionParam); putMsg(result, Status.SUCCESS); return result; } catch (Exception e) { String message = Optional.of(e).map(Throwable::getCause) .map(Throwable::getMessage) .orElse(e.getMessage()); - logger.error("Datasource test connection error, dbType:{}, connectionParam:{}, message:{}.", type, connectionParam, message); + logger.error("Datasource test connection error, dbType:{}, connectionParam:{}, message:{}.", type, + connectionParam, message); return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(), message); } } @@ -413,7 +426,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - return checkConnection(dataSource.getType(), DataSourceUtils.buildConnectionParams(dataSource.getType(), dataSource.getConnectionParams())); + return checkConnection(dataSource.getType(), + DataSourceUtils.buildConnectionParams(dataSource.getType(), dataSource.getConnectionParams())); } /** @@ -428,14 +442,15 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource public Result delete(User loginUser, int datasourceId) { Result result = new Result<>(); try { - //query datasource by id + // query datasource by id DataSource dataSource = dataSourceMapper.selectById(datasourceId); if (dataSource == null) { logger.warn("Datasource does not exist, datasourceId:{}.", datasourceId); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if (!canOperatorPermissions(loginUser, new Object[]{dataSource.getId()},AuthorizationType.DATASOURCE,DATASOURCE_DELETE)) { + if (!canOperatorPermissions(loginUser, new Object[]{dataSource.getId()}, AuthorizationType.DATASOURCE, + DATASOURCE_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -462,7 +477,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource public Map unauthDatasource(User loginUser, Integer userId) { Map result = new HashMap<>(); List datasourceList; - if (canOperatorPermissions(loginUser,null,AuthorizationType.DATASOURCE,null)) { + if (canOperatorPermissions(loginUser, null, AuthorizationType.DATASOURCE, null)) { // admin gets all data sources except userId datasourceList = dataSourceMapper.queryDatasourceExceptUserId(userId); } else { @@ -543,7 +558,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource tables = metaData.getTables( connectionParam.getDatabase(), - getDbSchemaPattern(dataSource.getType(),schema,connectionParam), + getDbSchemaPattern(dataSource.getType(), schema, connectionParam), "%", TABLE_TYPES); if (null == tables) { logger.error("Get datasource tables error, datasourceId:{}.", datasourceId); @@ -574,7 +589,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource } @Override - public Map getTableColumns(Integer datasourceId,String tableName) { + public Map getTableColumns(Integer datasourceId, String tableName) { Map result = new HashMap<>(); DataSource dataSource = dataSourceMapper.selectById(datasourceId); @@ -640,7 +655,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource return options; } - private String getDbSchemaPattern(DbType dbType,String schema,BaseConnectionParam connectionParam) { + private String getDbSchemaPattern(DbType dbType, String schema, BaseConnectionParam connectionParam) { if (dbType == null) { return null; } @@ -689,7 +704,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource } } } - private void clearBindTestId(Integer bindTestId){ + private void clearBindTestId(Integer bindTestId) { dataSourceMapper.clearBindTestId(bindTestId); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java index 417103f9f3..7d16d3d4d1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.DqExecuteResultService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.DqExecuteResult; import org.apache.dolphinscheduler.dao.entity.User; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java index 0f5efcfa3b..4023843c4b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -17,7 +17,9 @@ package org.apache.dolphinscheduler.api.service.impl; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_UPDATE; import org.apache.dolphinscheduler.api.dto.EnvironmentDto; import org.apache.dolphinscheduler.api.enums.Status; @@ -151,7 +153,8 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme relation.setCreateTime(new Date()); relation.setUpdateTime(new Date()); relationMapper.insert(relation); - logger.info("Environment-WorkerGroup relation create complete, environmentName:{}, workerGroup:{}.", + logger.info( + "Environment-WorkerGroup relation create complete, environmentName:{}, workerGroup:{}.", env.getName(), relation.getWorkerGroup()); } }); @@ -159,7 +162,8 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme } result.put(Constants.DATA_LIST, env.getCode()); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.ENVIRONMENT, loginUser.getId(), Collections.singletonList(env.getId()), logger); + permissionPostHandle(AuthorizationType.ENVIRONMENT, loginUser.getId(), + Collections.singletonList(env.getId()), logger); logger.info("Environment create complete, name:{}.", env.getName()); } else { logger.error("Environment create error, name:{}.", env.getName()); @@ -186,7 +190,8 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { environmentIPage = environmentMapper.queryEnvironmentListPaging(page, searchVal); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -199,12 +204,13 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme if (CollectionUtils.isNotEmpty(environmentIPage.getRecords())) { Map> relationMap = relationMapper.selectList(null).stream() - .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList()))); + .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode, + Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup, Collectors.toList()))); List dtoList = environmentIPage.getRecords().stream().map(environment -> { EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(environment,dto); - List workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList()); + BeanUtils.copyProperties(environment, dto); + List workerGroups = relationMap.getOrDefault(environment.getCode(), new ArrayList()); dto.setWorkerGroups(workerGroups); return dto; }).collect(Collectors.toList()); @@ -227,31 +233,33 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme */ @Override public Map queryAllEnvironmentList(User loginUser) { - Map result = new HashMap<>(); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, loginUser.getId(), logger); + Map result = new HashMap<>(); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, + loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); - putMsg(result,Status.SUCCESS); + putMsg(result, Status.SUCCESS); return result; } List environmentList = environmentMapper.selectBatchIds(ids); if (CollectionUtils.isNotEmpty(environmentList)) { Map> relationMap = relationMapper.selectList(null).stream() - .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList()))); + .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode, + Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup, Collectors.toList()))); List dtoList = environmentList.stream().map(environment -> { EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(environment,dto); - List workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList()); + BeanUtils.copyProperties(environment, dto); + List workerGroups = relationMap.getOrDefault(environment.getCode(), new ArrayList()); dto.setWorkerGroups(workerGroups); return dto; }).collect(Collectors.toList()); - result.put(Constants.DATA_LIST,dtoList); + result.put(Constants.DATA_LIST, dtoList); } else { result.put(Constants.DATA_LIST, new ArrayList<>()); } - putMsg(result,Status.SUCCESS); + putMsg(result, Status.SUCCESS); return result; } @@ -274,7 +282,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme .collect(Collectors.toList()); EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(env,dto); + BeanUtils.copyProperties(env, dto); dto.setWorkerGroups(workerGroups); result.put(Constants.DATA_LIST, dto); putMsg(result, Status.SUCCESS); @@ -300,7 +308,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme .collect(Collectors.toList()); EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(env,dto); + BeanUtils.copyProperties(env, dto); dto.setWorkerGroups(workerGroups); result.put(Constants.DATA_LIST, dto); putMsg(result, Status.SUCCESS); @@ -327,7 +335,8 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme .selectCount(new QueryWrapper().lambda().eq(TaskDefinition::getEnvironmentCode, code)); if (relatedTaskNumber > 0) { - logger.warn("Delete environment failed because {} tasks is using it, environmentCode:{}.", relatedTaskNumber, code); + logger.warn("Delete environment failed because {} tasks is using it, environmentCode:{}.", + relatedTaskNumber, code); putMsg(result, Status.DELETE_ENVIRONMENT_RELATED_TASK_EXISTS); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java index 7fa7104ebf..51e8fd458c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java @@ -36,8 +36,9 @@ import org.springframework.stereotype.Service; * task definition service impl */ @Service -public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl implements - EnvironmentWorkerGroupRelationService { +public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl + implements + EnvironmentWorkerGroupRelationService { private static final Logger logger = LoggerFactory.getLogger(EnvironmentWorkerGroupRelationServiceImpl.class); @@ -52,7 +53,8 @@ public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl i @Override public Map queryEnvironmentWorkerGroupRelation(Long environmentCode) { Map result = new HashMap<>(); - List relations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(environmentCode); + List relations = + environmentWorkerGroupRelationMapper.queryByEnvironmentCode(environmentCode); result.put(Constants.DATA_LIST, relations); putMsg(result, Status.SUCCESS); return result; @@ -69,8 +71,8 @@ public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl i List relations = environmentWorkerGroupRelationMapper.selectList(null); - result.put(Constants.DATA_LIST,relations); - putMsg(result,Status.SUCCESS); + result.put(Constants.DATA_LIST, relations); + putMsg(result, Status.SUCCESS); return result; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java index f04bf89d3f..21e3737101 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java @@ -23,13 +23,15 @@ import org.apache.dolphinscheduler.api.service.FavTaskService; import org.apache.dolphinscheduler.dao.entity.FavTask; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.FavTaskMapper; -import org.springframework.stereotype.Service; -import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; import java.util.Set; +import javax.annotation.Resource; + +import org.springframework.stereotype.Service; + @Service public class FavTaskServiceImpl extends BaseServiceImpl implements FavTaskService { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java index 97d4a67f2b..7316b3e337 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java @@ -31,8 +31,8 @@ import org.apache.dolphinscheduler.dao.mapper.ClusterMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Date; @@ -445,7 +445,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames if (isAdmin(loginUser)) { k8sNamespaces = k8sNamespaceMapper.selectList(null); } else { - k8sNamespaces = k8sNamespaceMapper.queryNamespaceAvailable(loginUser.getId()); + k8sNamespaces = k8sNamespaceMapper.queryNamespaceAvailable(loginUser.getId()); } setClusterName(k8sNamespaces); return k8sNamespaces; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index e3f83f3c1f..f12fc27cbb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.LoggerService; @@ -38,10 +41,6 @@ import org.apache.commons.lang3.StringUtils; import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.Objects; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,9 +49,6 @@ import org.springframework.stereotype.Service; import com.google.common.primitives.Bytes; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; - /** * logger service impl */ @@ -107,7 +103,6 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService return result; } - /** * get log size * @@ -137,7 +132,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService @SuppressWarnings("unchecked") public Map queryLog(User loginUser, long projectCode, int taskInstId, int skipLineNum, int limit) { Project project = projectMapper.queryByCode(projectCode); - //check user access for project + // check user access for project Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, VIEW_LOG); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; @@ -170,8 +165,8 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService @Override public byte[] getLogBytes(User loginUser, long projectCode, int taskInstId) { Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode,DOWNLOAD_LOG); + // check user access for project + Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, DOWNLOAD_LOG); if (result.get(Constants.STATUS) != Status.SUCCESS) { throw new ServiceException("user has no permission"); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java index 04debec555..fb10f49cb1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java @@ -97,32 +97,30 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic Map result = new HashMap<>(); List workerServers = getServerListFromRegistry(false) - .stream() - .map((Server server) -> { - WorkerServerModel model = new WorkerServerModel(); - model.setId(server.getId()); - model.setHost(server.getHost()); - model.setPort(server.getPort()); - model.setZkDirectories(Sets.newHashSet(server.getZkDirectory())); - model.setResInfo(server.getResInfo()); - model.setCreateTime(server.getCreateTime()); - model.setLastHeartbeatTime(server.getLastHeartbeatTime()); - return model; - }) - .collect(Collectors.toList()); + .stream() + .map((Server server) -> { + WorkerServerModel model = new WorkerServerModel(); + model.setId(server.getId()); + model.setHost(server.getHost()); + model.setPort(server.getPort()); + model.setZkDirectories(Sets.newHashSet(server.getZkDirectory())); + model.setResInfo(server.getResInfo()); + model.setCreateTime(server.getCreateTime()); + model.setLastHeartbeatTime(server.getLastHeartbeatTime()); + return model; + }) + .collect(Collectors.toList()); Map workerHostPortServerMapping = workerServers - .stream() - .collect(Collectors.toMap( - (WorkerServerModel worker) -> { - String[] s = worker.getZkDirectories().iterator().next().split("/"); - return s[s.length - 1]; - } - , Function.identity() - , (WorkerServerModel oldOne, WorkerServerModel newOne) -> { - oldOne.getZkDirectories().addAll(newOne.getZkDirectories()); - return oldOne; - })); + .stream() + .collect(Collectors.toMap( + (WorkerServerModel worker) -> { + String[] s = worker.getZkDirectories().iterator().next().split("/"); + return s[s.length - 1]; + }, Function.identity(), (WorkerServerModel oldOne, WorkerServerModel newOne) -> { + oldOne.getZkDirectories().addAll(newOne.getZkDirectories()); + return oldOne; + })); result.put(Constants.DATA_LIST, workerHostPortServerMapping.values()); putMsg(result, Status.SUCCESS); @@ -133,8 +131,8 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic @Override public List getServerListFromRegistry(boolean isMaster) { return isMaster - ? registryClient.getServerList(NodeType.MASTER) - : registryClient.getServerList(NodeType.WORKER); + ? registryClient.getServerList(NodeType.MASTER) + : registryClient.getServerList(NodeType.WORKER); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 37659730d1..17979139d5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -973,8 +973,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro diffCode.stream().map(String::valueOf).collect(Collectors.joining(Constants.COMMA))); throw new ServiceException(Status.BATCH_DELETE_PROCESS_DEFINE_BY_CODES_ERROR, diffCode.stream().map(code -> code + "[process definition not exist]") - .collect(Collectors.joining(Constants.COMMA)) - ); + .collect(Collectors.joining(Constants.COMMA))); } for (ProcessDefinition process : processDefinitionList) { @@ -1003,7 +1002,8 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // check process instances is already running List processInstances = processInstanceService - .queryByProcessDefineCodeAndStatus(processDefinition.getCode(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); + .queryByProcessDefineCodeAndStatus(processDefinition.getCode(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); if (CollectionUtils.isNotEmpty(processInstances)) { throw new ServiceException(Status.DELETE_PROCESS_DEFINITION_EXECUTING_FAIL, processInstances.size()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 4f146a8c2f..b969ea6dcc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -17,10 +17,20 @@ package org.apache.dolphinscheduler.api.service.impl; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_UPDATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_INSTANCE; +import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_NOT_EXIST; +import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR; +import static org.apache.dolphinscheduler.common.Constants.DATA_LIST; +import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; +import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; +import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; + import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -40,7 +50,6 @@ import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -66,16 +75,13 @@ import org.apache.dolphinscheduler.dao.utils.WorkflowUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.StringUtils; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -92,20 +98,15 @@ import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_DELETE; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_UPDATE; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_INSTANCE; -import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_NOT_EXIST; -import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR; -import static org.apache.dolphinscheduler.common.Constants.DATA_LIST; -import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; -import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS; -import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; -import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; -import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * process instance service impl @@ -348,20 +349,24 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce public Map queryTaskListByProcessId(User loginUser, long projectCode, Integer processId) throws IOException { Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE); + // check user access for project + Map result = + projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; } ProcessInstance processInstance = processService.findProcessInstanceDetailById(processId) .orElseThrow(() -> new ServiceException(PROCESS_INSTANCE_NOT_EXIST, processId)); - ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); + ProcessDefinition processDefinition = + processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); if (processDefinition != null && projectCode != processDefinition.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionId:{}.", projectCode, processId); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionId:{}.", projectCode, + processId); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processId); return result; } - List taskInstanceList = processService.findValidTaskListByProcessId(processId, processInstance.getTestFlag()); + List taskInstanceList = + processService.findValidTaskListByProcessId(processId, processInstance.getTestFlag()); addDependResultForTaskList(taskInstanceList); Map resultMap = new HashMap<>(); resultMap.put(PROCESS_INSTANCE_STATE, processInstance.getState().toString()); @@ -378,7 +383,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce private void addDependResultForTaskList(List taskInstanceList) throws IOException { for (TaskInstance taskInstance : taskInstanceList) { if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) { - logger.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}", taskInstance.getTaskCode(), taskInstance.getId()); + logger.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}", + taskInstance.getTaskCode(), taskInstance.getId()); Result logResult = loggerService.queryLog( taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT); if (logResult.getCode() == Status.SUCCESS.ordinal()) { @@ -447,13 +453,15 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskInstance.getTaskCode()); if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - logger.error("Task definition does not exist, projectCode:{}, taskDefinitionCode:{}.", projectCode, taskInstance.getTaskCode()); + logger.error("Task definition does not exist, projectCode:{}, taskDefinitionCode:{}.", projectCode, + taskInstance.getTaskCode()); putMsg(result, Status.TASK_INSTANCE_NOT_EXISTS, taskId); return result; } if (!taskInstance.isSubProcess()) { - logger.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.", TASK_TYPE_SUB_PROCESS, projectCode, taskId); + logger.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.", + TASK_TYPE_SUB_PROCESS, projectCode, taskId); putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName()); return result; } @@ -461,7 +469,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessInstance subWorkflowInstance = processService.findSubProcessInstance( taskInstance.getProcessInstanceId(), taskInstance.getId()); if (subWorkflowInstance == null) { - logger.error("SubProcess instance does not exist, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstance.getId()); + logger.error("SubProcess instance does not exist, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstance.getId()); putMsg(result, Status.SUB_PROCESS_INSTANCE_NOT_EXIST, taskId); return result; } @@ -496,8 +505,9 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce String globalParams, String locations, int timeout, String tenantCode) { Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, INSTANCE_UPDATE); + // check user access for project + Map result = + projectService.checkProjectAndAuth(loginUser, project, projectCode, INSTANCE_UPDATE); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; } @@ -505,15 +515,18 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessInstance processInstance = processService.findProcessInstanceDetailById(processInstanceId) .orElseThrow(() -> new ServiceException(PROCESS_INSTANCE_NOT_EXIST, processInstanceId)); // check process instance exists in project - ProcessDefinition processDefinition0 = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); + ProcessDefinition processDefinition0 = + processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); if (processDefinition0 != null && projectCode != processDefinition0.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, processInstance.getProcessDefinitionCode()); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, + processInstance.getProcessDefinitionCode()); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } - //check process instance status + // check process instance status if (!processInstance.getState().isFinished()) { - logger.warn("Process Instance state is {} so can not update process instance, processInstanceId:{}.", processInstance.getState().getDesc(), processInstanceId); + logger.warn("Process Instance state is {} so can not update process instance, processInstanceId:{}.", + processInstance.getState().getDesc(), processInstanceId); putMsg(result, PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstance.getName(), processInstance.getState().toString(), "update"); return result; @@ -548,7 +561,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce } int saveTaskResult = processService.saveTaskDefine(loginUser, projectCode, taskDefinitionLogs, syncDefine); if (saveTaskResult == Constants.DEFINITION_FAILURE) { - logger.error("Update task definition error, projectCode:{}, processInstanceId:{}", projectCode, processInstanceId); + logger.error("Update task definition error, projectCode:{}, processInstanceId:{}", projectCode, + processInstanceId); putMsg(result, Status.UPDATE_TASK_DEFINITION_ERROR); throw new ServiceException(Status.UPDATE_TASK_DEFINITION_ERROR); } @@ -576,20 +590,24 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce processDefinition.setUpdateTime(new Date()); int insertVersion = processService.saveProcessDefine(loginUser, processDefinition, syncDefine, Boolean.FALSE); if (insertVersion == 0) { - logger.error("Update process definition error, projectCode:{}, processDefinitionName:{}.", projectCode, processDefinition.getName()); + logger.error("Update process definition error, projectCode:{}, processDefinitionName:{}.", projectCode, + processDefinition.getName()); putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); throw new ServiceException(Status.UPDATE_PROCESS_DEFINITION_ERROR); } else - logger.info("Update process definition complete, projectCode:{}, processDefinitionName:{}.", projectCode, processDefinition.getName()); + logger.info("Update process definition complete, projectCode:{}, processDefinitionName:{}.", projectCode, + processDefinition.getName()); int insertResult = processService.saveTaskRelation(loginUser, processDefinition.getProjectCode(), processDefinition.getCode(), insertVersion, taskRelationList, taskDefinitionLogs, syncDefine); if (insertResult == Constants.EXIT_CODE_SUCCESS) { - logger.info("Update task relations complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", + logger.info( + "Update task relations complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", projectCode, processDefinition.getCode(), insertVersion); putMsg(result, Status.SUCCESS); result.put(Constants.DATA_LIST, processDefinition); } else { - logger.info("Update task relations error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", + logger.info( + "Update task relations error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", projectCode, processDefinition.getCode(), insertVersion); putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); throw new ServiceException(Status.UPDATE_PROCESS_DEFINITION_ERROR); @@ -597,12 +615,14 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce processInstance.setProcessDefinitionVersion(insertVersion); int update = processInstanceDao.updateProcessInstance(processInstance); if (update == 0) { - logger.error("Update process instance version error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}", + logger.error( + "Update process instance version error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}", projectCode, processDefinition.getCode(), insertVersion); putMsg(result, Status.UPDATE_PROCESS_INSTANCE_ERROR); throw new ServiceException(Status.UPDATE_PROCESS_INSTANCE_ERROR); } - logger.info("Update process instance complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}, processInstanceId:{}", + logger.info( + "Update process instance complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}, processInstanceId:{}", projectCode, processDefinition.getCode(), insertVersion, processInstanceId); putMsg(result, Status.SUCCESS); return result; @@ -649,14 +669,17 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessInstance subInstance = processService.findProcessInstanceDetailById(subId) .orElseThrow(() -> new ServiceException(PROCESS_INSTANCE_NOT_EXIST, subId)); if (subInstance.getIsSubProcess() == Flag.NO) { - logger.warn("Process instance is not sub process instance type, processInstanceId:{}, processInstanceName:{}.", subId, subInstance.getName()); + logger.warn( + "Process instance is not sub process instance type, processInstanceId:{}, processInstanceName:{}.", + subId, subInstance.getName()); putMsg(result, Status.PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE, subInstance.getName()); return result; } ProcessInstance parentWorkflowInstance = processService.findParentProcessInstance(subId); if (parentWorkflowInstance == null) { - logger.error("Parent process instance does not exist, projectCode:{}, subProcessInstanceId:{}.", projectCode, subId); + logger.error("Parent process instance does not exist, projectCode:{}, subProcessInstanceId:{}.", + projectCode, subId); putMsg(result, Status.SUB_PROCESS_INSTANCE_NOT_EXIST); return result; } @@ -691,7 +714,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce if (!processInstance.getState().isFinished()) { logger.warn("Process Instance state is {} so can not delete process instance, processInstanceId:{}.", processInstance.getState().getDesc(), processInstanceId); - throw new ServiceException(PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstance.getName(), processInstance.getState(), "delete"); + throw new ServiceException(PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstance.getName(), + processInstance.getState(), "delete"); } ProcessDefinition processDefinition = @@ -706,7 +730,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce processService.removeTaskLogFile(processInstanceId); } catch (Exception ignore) { // ignore - logger.warn("Remove task log file exception, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", + logger.warn( + "Remove task log file exception, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", projectCode, processInstance.getProcessDefinitionCode(), processInstanceId); } @@ -718,11 +743,13 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce processService.deleteWorkTaskInstanceByProcessInstanceId(processInstanceId); if (delete > 0) { - logger.info("Delete process instance complete, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", + logger.info( + "Delete process instance complete, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", projectCode, processInstance.getProcessDefinitionCode(), processInstanceId); putMsg(result, Status.SUCCESS); } else { - logger.error("Delete process instance error, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", + logger.error( + "Delete process instance error, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", projectCode, processInstance.getProcessDefinitionCode(), processInstanceId); putMsg(result, Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR); throw new ServiceException(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR); @@ -745,7 +772,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId); if (processInstance == null) { - logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, + processInstanceId); putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } @@ -753,7 +781,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); if (processDefinition != null && projectCode != processDefinition.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, processInstance.getProcessDefinitionCode()); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, + processInstance.getProcessDefinitionCode()); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } @@ -801,7 +830,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce Map timeParams) { Map> localUserDefParams = new HashMap<>(); List taskInstanceList = - taskInstanceMapper.findValidTaskListByProcessId(processInstance.getId(), Flag.YES, processInstance.getTestFlag()); + taskInstanceMapper.findValidTaskListByProcessId(processInstance.getId(), Flag.YES, + processInstance.getTestFlag()); for (TaskInstance taskInstance : taskInstanceList) { TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMapper.queryByDefinitionCodeAndVersion( taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()); @@ -836,7 +866,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId); if (processInstance == null) { - logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, + processInstanceId); putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } @@ -845,7 +876,8 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce processInstance.getProcessDefinitionCode(), processInstance.getProcessDefinitionVersion()); if (processDefinition == null || projectCode != processDefinition.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, processInstance.getProcessDefinitionCode()); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, + processInstance.getProcessDefinitionCode()); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index bd43e549b4..401ec6abd2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -189,7 +189,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic public Map queryByName(User loginUser, String projectName) { Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); - boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result,PROJECT); + boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result, PROJECT); if (!hasProjectAndPerm) { return result; } @@ -218,7 +218,8 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic } else if (!canOperatorPermissions(loginUser, new Object[]{project.getId()}, AuthorizationType.PROJECTS, permission)) { // check read permission - logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", permission, loginUser.getUserName(), projectCode); + logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", + permission, loginUser.getUserName(), projectCode); putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), projectCode); } else { putMsg(result, Status.SUCCESS); @@ -244,9 +245,10 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic if (project == null) { logger.error("Project does not exist."); putMsg(result, Status.PROJECT_NOT_FOUND, ""); - } else if (!canOperatorPermissions(loginUser, new Object[] {project.getId()}, AuthorizationType.PROJECTS, + } else if (!canOperatorPermissions(loginUser, new Object[]{project.getId()}, AuthorizationType.PROJECTS, permission)) { - logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", permission, loginUser.getUserName(), project.getCode()); + logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", + permission, loginUser.getUserName(), project.getCode()); putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getCode()); } else { checkResult = true; @@ -260,9 +262,10 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic if (project == null) { logger.error("Project does not exist."); putMsg(result, Status.PROJECT_NOT_FOUND, ""); - } else if (!canOperatorPermissions(loginUser, new Object[] {project.getId()}, AuthorizationType.PROJECTS, + } else if (!canOperatorPermissions(loginUser, new Object[]{project.getId()}, AuthorizationType.PROJECTS, permission)) { - logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", permission, loginUser.getUserName(), project.getCode()); + logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", + permission, loginUser.getUserName(), project.getCode()); putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName()); } else { checkResult = true; @@ -353,8 +356,9 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic * @param project project * @return check result */ - private Map getCheckResult(User loginUser, Project project,String perm) { - Map checkResult = checkProjectAndAuth(loginUser, project, project == null ? 0L : project.getCode(),perm); + private Map getCheckResult(User loginUser, Project project, String perm) { + Map checkResult = + checkProjectAndAuth(loginUser, project, project == null ? 0L : project.getCode(), perm); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { return checkResult; @@ -413,7 +417,6 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic return result; } - /** * query unauthorized project * @@ -425,13 +428,16 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic public Result queryUnauthorizedProject(User loginUser, Integer userId) { Result result = new Result(); - Set projectIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); if (projectIds.isEmpty()) { result.setData(Collections.emptyList()); putMsg(result, Status.SUCCESS); return result; } - List projectList = projectMapper.listAuthorizedProjects(loginUser.getUserType().equals(UserType.ADMIN_USER) ? 0 : loginUser.getId(), new ArrayList<>(projectIds)); + List projectList = projectMapper.listAuthorizedProjects( + loginUser.getUserType().equals(UserType.ADMIN_USER) ? 0 : loginUser.getId(), + new ArrayList<>(projectIds)); List resultList = new ArrayList<>(); Set projectSet; @@ -535,7 +541,8 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic public Result queryProjectCreatedAndAuthorizedByUser(User loginUser) { Result result = new Result(); - Set projectIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); if (projectIds.isEmpty()) { result.setData(Collections.emptyList()); putMsg(result, Status.SUCCESS); @@ -595,7 +602,8 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic @Override public Result queryAllProjectList(User user) { Result result = new Result(); - List projects = projectMapper.queryAllProject(user.getUserType() == UserType.ADMIN_USER ? 0 : user.getId()); + List projects = + projectMapper.queryAllProject(user.getUserType() == UserType.ADMIN_USER ? 0 : user.getId()); result.setData(projects); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java index 0bb8bd4284..bfba4d08d7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java @@ -37,10 +37,8 @@ import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; @@ -91,7 +89,8 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { * @param updateQueue The queue object want to update */ private void updateQueueValid(Queue existsQueue, Queue updateQueue) throws ServiceException { - // Check the exists queue and the necessary of update operation, in not exist checker have to use updateQueue to avoid NPE + // Check the exists queue and the necessary of update operation, in not exist checker have to use updateQueue to + // avoid NPE if (Objects.isNull(existsQueue)) { throw new ServiceException(Status.QUEUE_NOT_EXIST, updateQueue.getQueue()); } else if (Objects.equals(existsQueue, updateQueue)) { @@ -102,9 +101,11 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.QUEUE); } else if (StringUtils.isEmpty(updateQueue.getQueueName())) { throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.QUEUE_NAME); - } else if (!Objects.equals(updateQueue.getQueue(), existsQueue.getQueue()) && checkQueueExist(updateQueue.getQueue())) { + } else if (!Objects.equals(updateQueue.getQueue(), existsQueue.getQueue()) + && checkQueueExist(updateQueue.getQueue())) { throw new ServiceException(Status.QUEUE_VALUE_EXIST, updateQueue.getQueue()); - } else if (!Objects.equals(updateQueue.getQueueName(), existsQueue.getQueueName()) && checkQueueNameExist(updateQueue.getQueueName())) { + } else if (!Objects.equals(updateQueue.getQueueName(), existsQueue.getQueueName()) + && checkQueueNameExist(updateQueue.getQueueName())) { throw new ServiceException(Status.QUEUE_NAME_EXIST, updateQueue.getQueueName()); } } @@ -118,7 +119,8 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { @Override public Result queryList(User loginUser) { Result result = new Result(); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, + loginUser.getId(), logger); if (loginUser.getUserType().equals(UserType.GENERAL_USER)) { ids = ids.isEmpty() ? new HashSet<>() : ids; ids.add(Constants.DEFAULT_QUEUE_ID); @@ -142,7 +144,8 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { public Result queryList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Result result = new Result(); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, + loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -171,7 +174,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { @Transactional public Result createQueue(User loginUser, String queue, String queueName) { Result result = new Result(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.QUEUE,YARN_QUEUE_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.QUEUE, YARN_QUEUE_CREATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -182,7 +185,8 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { result.setData(queueObj); logger.info("Queue create complete, queueName:{}.", queueObj.getQueueName()); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.QUEUE, loginUser.getId(), Collections.singletonList(queueObj.getId()), logger); + permissionPostHandle(AuthorizationType.QUEUE, loginUser.getId(), Collections.singletonList(queueObj.getId()), + logger); return result; } @@ -198,7 +202,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { @Override public Result updateQueue(User loginUser, int id, String queue, String queueName) { Result result = new Result(); - if (!canOperatorPermissions(loginUser,new Object[]{id}, AuthorizationType.QUEUE,YARN_QUEUE_UPDATE)) { + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.QUEUE, YARN_QUEUE_UPDATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -208,8 +212,9 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { // check old queue using by any user if (checkIfQueueIsInUsing(existsQueue.getQueueName(), updateQueue.getQueueName())) { - //update user related old queue - Integer relatedUserNums = userMapper.updateUserQueue(existsQueue.getQueueName(), updateQueue.getQueueName()); + // update user related old queue + Integer relatedUserNums = + userMapper.updateUserQueue(existsQueue.getQueueName(), updateQueue.getQueueName()); logger.info("Old queue have related {} users, exec update user success.", relatedUserNums); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index abb8135ad4..55e7eaaab1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -279,7 +279,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } if (fullName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) { - logger.warn("Resource file's name is longer than max full name length, fullName:{}, fullNameSize:{}, maxFullNameSize:{}", + logger.warn( + "Resource file's name is longer than max full name length, fullName:{}, fullNameSize:{}, maxFullNameSize:{}", RegexUtils.escapeNRT(name), fullName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH); putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR); return result; @@ -311,7 +312,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe logger.error("Upload resource file failed, resourceName:{}, fileName:{}.", RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - throw new ServiceException(String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); + throw new ServiceException( + String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); } else logger.info("Upload resource file complete, resourceName:{}, fileName:{}.", RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename())); @@ -340,7 +342,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe parentResource.setSize(0L); } resourcesMapper.updateById(parentResource); - logger.info("Resource size update complete, resourceFullName:{}, newSize:{}.", parentResource.getFullName(), parentResource.getSize()); + logger.info("Resource size update complete, resourceFullName:{}, newSize:{}.", + parentResource.getFullName(), parentResource.getSize()); } } } @@ -405,7 +408,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.STORAGE_NOT_STARTUP); return result; } @@ -452,7 +456,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe String originFileName = storageOperate.getFileName(resource.getType(), tenantCode, originFullName); try { if (!storageOperate.exists(tenantCode, originFileName)) { - logger.error("Resource file does not exist in {} storage, tenantCode:{}, resourceId:{}, originFileName:{}.", + logger.error( + "Resource file does not exist in {} storage, tenantCode:{}, resourceId:{}, originFileName:{}.", resource.getType(), tenantCode, resourceId, originFileName); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; @@ -613,7 +618,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) { - logger.warn("File original name verify failed, fileOriginalName:{}.", RegexUtils.escapeNRT(file.getOriginalFilename())); + logger.warn("File original name verify failed, fileOriginalName:{}.", + RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); return result; } @@ -621,7 +627,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (file != null) { // file is empty if (file.isEmpty()) { - logger.warn("Parameter file is empty, fileOriginalName:{}.", RegexUtils.escapeNRT(file.getOriginalFilename())); + logger.warn("Parameter file is empty, fileOriginalName:{}.", + RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.RESOURCE_FILE_IS_EMPTY); return result; } @@ -646,7 +653,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } if (file.getSize() > Constants.MAX_FILE_SIZE) { - logger.warn("Resource file size is larger than max file size, fileOriginalName:{}, fileSize:{}, maxFileSize:{}.", + logger.warn( + "Resource file size is larger than max file size, fileOriginalName:{}, fileSize:{}, maxFileSize:{}.", RegexUtils.escapeNRT(file.getOriginalFilename()), file.getSize(), Constants.MAX_FILE_SIZE); putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT); return result; @@ -876,7 +884,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe Integer[] needDeleteResourceIdArray = allChildren.toArray(new Integer[allChildren.size()]); if (needDeleteResourceIdArray.length >= 2) { - logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.", resource.getFileName()); + logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.", + resource.getFileName()); putMsg(result, Status.RESOURCE_HAS_FOLDER, resource.getFileName()); return result; } @@ -886,14 +895,16 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe List udfFuncs = udfFunctionMapper.listUdfByResourceId(needDeleteResourceIdArray); List udfFuncIds = udfFuncs.stream().map(UdfFunc::getId).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(udfFuncs)) { - logger.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", udfFuncIds); + logger.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", + udfFuncIds); putMsg(result, Status.UDF_RESOURCE_IS_BOUND, udfFuncs.get(0).getFuncName()); return result; } } if (resourceIdSet.contains(resource.getPid())) { - logger.warn("Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", + logger.warn( + "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", resource.getId(), resource.getPid()); putMsg(result, Status.RESOURCE_IS_USED); return result; @@ -901,7 +912,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe resourceIdSet.retainAll(allChildren); if (CollectionUtils.isNotEmpty(resourceIdSet)) { for (Integer resId : resourceIdSet) { - logger.warn("Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", + logger.warn( + "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", resId, resourceProcessMap.get(resId)); } putMsg(result, Status.RESOURCE_IS_USED); @@ -959,7 +971,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe try { String filename = storageOperate.getFileName(type, tenantCode, fullName); if (storageOperate.exists(tenantCode, filename)) { - logger.warn("Resource file with same name exists so can not create again, tenantCode:{}, resourceName:{}.", + logger.warn( + "Resource file with same name exists so can not create again, tenantCode:{}, resourceName:{}.", tenantCode, RegexUtils.escapeNRT(filename)); putMsg(result, Status.RESOURCE_FILE_EXIST, filename); } @@ -969,7 +982,9 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); } } else { - logger.error("Tenant does not exist or tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName()); + logger.error( + "Tenant does not exist or tenant of current login user does not specified, loginUserName:{}.", + loginUser.getUserName()); putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); } } @@ -1114,7 +1129,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (storageOperate.exists(tenantCode, resourceFileName)) { List content = storageOperate.vimFile(tenantCode, resourceFileName, skipLineNum, limit); - logger.info("Vim file content in path {} success, tenantCode:{}, fileName:{}, skipLineNum:{}, limit:{}.", + logger.info( + "Vim file content in path {} success, tenantCode:{}, fileName:{}, skipLineNum:{}, limit:{}.", resourceFileName, tenantCode, resourceFileName, skipLineNum, limit); putMsg(result, Status.SUCCESS); Map map = new HashMap<>(); @@ -1325,7 +1341,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe putMsg(result, Status.SUCCESS); // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.STORAGE_NOT_STARTUP); return result; } @@ -1437,7 +1454,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (!FileUtils.writeContent2File(content, localFilename)) { // write file fail - logger.error("Write file error, fileName:{}, content:{}.", localFilename, RegexUtils.escapeNRT(content)); + logger.error("Write file error, fileName:{}, content:{}.", localFilename, + RegexUtils.escapeNRT(content)); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } @@ -1450,7 +1468,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (!storageOperate.exists(tenantCode, resourcePath)) { // create if tenant dir not exists storageOperate.createTenantDirIfNotExists(tenantCode); - logger.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, tenantCode); + logger.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, + tenantCode); } if (storageOperate.exists(tenantCode, storageFileName)) { storageOperate.delete(tenantCode, storageFileName, false); @@ -1458,7 +1477,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe storageOperate.upload(tenantCode, localFilename, storageFileName, true, true); } catch (Exception e) { - logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, storageFileName, e); + logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, + storageFileName, e); result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, storageFileName)); return result; @@ -1479,7 +1499,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe public org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException { // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.warn("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.warn("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); throw new ServiceException("hdfs not startup"); } @@ -1514,7 +1535,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe Tenant tenant = tenantMapper.queryById(user.getTenantId()); if (tenant == null) { logger.error("Tenant does not exists, tenantId:{}.", user.getTenantId()); - throw new ServiceException(String.format("The tenant id %d of resource owner does not exist", user.getTenantId())); + throw new ServiceException( + String.format("The tenant id %d of resource owner does not exist", user.getTenantId())); } String tenantCode = tenant.getTenantCode(); @@ -1526,7 +1548,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe try { storageOperate.download(tenantCode, fileName, localFileName, false, true); - org.springframework.core.io.Resource file2Resource = org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); + org.springframework.core.io.Resource file2Resource = + org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); if (file2Resource != null) { logger.info("Download resource complete, path:{}, localFileName:{}.", fileName, localFileName); } @@ -1578,7 +1601,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe User user = userMapper.queryByUserNameAccurately(userName); Result resourceResponse = this.queryResource(user, fullName, null, ResourceType.FILE); if (resourceResponse.getCode() != Status.SUCCESS.getCode()) { - String msg = String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fullName); + String msg = + String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fullName); logger.error(msg); throw new IllegalArgumentException(msg); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java index 6734a21990..c8439ca6b2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java @@ -153,7 +153,7 @@ public class SessionServiceImpl extends BaseServiceImpl implements SessionServic */ Session session = sessionMapper.queryByUserIdAndIp(loginUser.getId(), ip); - //delete session + // delete session sessionMapper.deleteById(session.getId()); } catch (Exception e) { logger.warn("userId : {} , ip : {} , find more one session", loginUser.getId(), ip, e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java index c9c1eba5af..c96996e3bc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java @@ -65,20 +65,21 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr * @return tasks list */ @Override - public Map queryTasksByGroupId(User loginUser, String taskName - , String processName, Integer status, int groupId, int pageNo, int pageSize) { + public Map queryTasksByGroupId(User loginUser, String taskName, String processName, Integer status, + int groupId, int pageNo, int pageSize) { Map result = new HashMap<>(); Page page = new Page<>(pageNo, pageSize); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set projectIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); if (projectIds.isEmpty()) { result.put(Constants.DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; } List projects = projectMapper.selectBatchIds(projectIds); - IPage taskGroupQueue = taskGroupQueueMapper.queryTaskGroupQueueByTaskGroupIdPaging(page, taskName - ,processName,status,groupId,projects); + IPage taskGroupQueue = taskGroupQueueMapper.queryTaskGroupQueueByTaskGroupIdPaging(page, + taskName, processName, status, groupId, projects); pageInfo.setTotal((int) taskGroupQueue.getTotal()); pageInfo.setTotalList(taskGroupQueue.getRecords()); @@ -99,7 +100,7 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr */ @Override public Map queryTasksByProcessId(User loginUser, int pageNo, int pageSize, int processId) { - return this.doQuery(loginUser, pageNo, pageSize, processId); + return this.doQuery(loginUser, pageNo, pageSize, processId); } /** @@ -112,7 +113,7 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr */ @Override public Map queryAllTasks(User loginUser, int pageNo, int pageSize) { - return this.doQuery(loginUser, pageNo, pageSize, 0); + return this.doQuery(loginUser, pageNo, pageSize, 0); } public Map doQuery(User loginUser, int pageNo, int pageSize, @@ -145,12 +146,12 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr } @Override - public void forceStartTask(int queueId,int forceStart) { - taskGroupQueueMapper.updateForceStart(queueId,forceStart); + public void forceStartTask(int queueId, int forceStart) { + taskGroupQueueMapper.updateForceStart(queueId, forceStart); } @Override public void modifyPriority(Integer queueId, Integer priority) { - taskGroupQueueMapper.modifyPriority(queueId,priority); + taskGroupQueueMapper.modifyPriority(queueId, priority); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java index d954b94413..6f22af8662 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java @@ -117,7 +117,8 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe taskGroup.setCreateTime(new Date()); taskGroup.setUpdateTime(new Date()); if (taskGroupMapper.insert(taskGroup) > 0) { - permissionPostHandle(AuthorizationType.TASK_GROUP, loginUser.getId(), Collections.singletonList(taskGroup.getId()),logger); + permissionPostHandle(AuthorizationType.TASK_GROUP, loginUser.getId(), + Collections.singletonList(taskGroup.getId()), logger); logger.info("Create task group complete, taskGroupName:{}.", taskGroup.getName()); putMsg(result, Status.SUCCESS); } else { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index 1aabfd44c7..7c6b724c90 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -38,11 +38,11 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskSavePointRequestCommand; import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; import java.util.Date; @@ -154,12 +154,12 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst if (taskExecuteType == TaskExecuteType.STREAM) { // stream task without process instance taskInstanceIPage = taskInstanceMapper.queryStreamTaskInstanceListPaging( - page, project.getCode(), processDefinitionName, searchVal, taskName, executorId, statusArray, host, taskExecuteType, start, end - ); + page, project.getCode(), processDefinitionName, searchVal, taskName, executorId, statusArray, host, + taskExecuteType, start, end); } else { taskInstanceIPage = taskInstanceMapper.queryTaskInstanceListPaging( - page, project.getCode(), processInstanceId, processInstanceName, searchVal, taskName, executorId, statusArray, host, taskExecuteType, start, end - ); + page, project.getCode(), processInstanceId, processInstanceName, searchVal, taskName, executorId, + statusArray, host, taskExecuteType, start, end); } Set exclusionSet = new HashSet<>(); exclusionSet.add(Constants.CLASS); @@ -205,21 +205,24 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst // check whether the task instance can be found TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); if (task == null) { - logger.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstanceId); + logger.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstanceId); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(task.getTaskCode()); if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - logger.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, task.getTaskCode()); + logger.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, + task.getTaskCode()); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND, taskInstanceId); return result; } // check whether the task instance state type is failure or cancel if (!task.getState().isFailure() && !task.getState().isKill()) { - logger.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", task.getState().getDesc(), projectCode, taskInstanceId); + logger.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", + task.getState().getDesc(), projectCode, taskInstanceId); putMsg(result, Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState().toString()); return result; } @@ -229,10 +232,12 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst int changedNum = taskInstanceMapper.updateById(task); if (changedNum > 0) { processService.forceProcessInstanceSuccessByTaskInstanceId(taskInstanceId); - logger.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, taskInstanceId); + logger.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, + taskInstanceId); putMsg(result, Status.SUCCESS); } else { - logger.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, taskInstanceId); + logger.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", + projectCode, taskInstanceId); putMsg(result, Status.FORCE_TASK_SUCCESS_ERROR); } return result; @@ -243,17 +248,19 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst Result result = new Result(); Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); + // check user access for project + Map checkResult = + projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { - putMsg(result,status); + putMsg(result, status); return result; } TaskInstance taskInstance = taskInstanceMapper.selectById(taskInstanceId); if (taskInstance == null) { - logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstanceId); + logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstanceId); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } @@ -272,17 +279,19 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst Result result = new Result(); Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectCode,FORCED_SUCCESS); + // check user access for project + Map checkResult = + projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { - putMsg(result,status); + putMsg(result, status); return result; } TaskInstance taskInstance = taskInstanceMapper.selectById(taskInstanceId); if (taskInstance == null) { - logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstanceId); + logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstanceId); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 169b0a6348..682f70c296 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.api.service.impl; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TENANT_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TENANT_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TENANT_UPDATE; +import static org.apache.dolphinscheduler.common.Constants.TENANT_FULL_NAME_MAX_LENGTH; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.QueueService; @@ -41,23 +42,26 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.service.storage.StorageOperate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; -import static org.apache.dolphinscheduler.common.Constants.TENANT_FULL_NAME_MAX_LENGTH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * tenant service impl @@ -121,7 +125,8 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService throw new ServiceException(Status.TENANT_FULL_NAME_TOO_LONG_ERROR); } else if (!RegexUtils.isValidLinuxUserName(updateTenant.getTenantCode())) { throw new ServiceException(Status.CHECK_OS_TENANT_CODE_ERROR); - } else if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) && checkTenantExists(updateTenant.getTenantCode())) { + } else if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) + && checkTenantExists(updateTenant.getTenantCode())) { throw new ServiceException(Status.OS_TENANT_CODE_EXIST, updateTenant.getTenantCode()); } } @@ -144,10 +149,10 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService String desc) throws Exception { Map result = new HashMap<>(); result.put(Constants.STATUS, false); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT, TENANT_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_CREATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -160,7 +165,8 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService if (PropertyUtils.getResUploadStartupState()) { storageOperate.createTenantDirIfNotExists(tenantCode); } - permissionPostHandle(AuthorizationType.TENANT, loginUser.getId(), Collections.singletonList(tenant.getId()), logger); + permissionPostHandle(AuthorizationType.TENANT, loginUser.getId(), Collections.singletonList(tenant.getId()), + logger); result.put(Constants.DATA_LIST, tenant); putMsg(result, Status.SUCCESS); return result; @@ -180,7 +186,8 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService Result result = new Result<>(); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, + loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -213,10 +220,10 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT,TENANT_UPDATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_UPDATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -229,7 +236,8 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService /** * if the tenant code is modified, the original resource needs to be copied to the new tenant. */ - if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) && PropertyUtils.getResUploadStartupState()) { + if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) + && PropertyUtils.getResUploadStartupState()) { storageOperate.createTenantDirIfNotExists(tenantCode); } int update = tenantMapper.updateById(updateTenant); @@ -256,7 +264,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService public Map deleteTenantById(User loginUser, int id) throws Exception { Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT,TENANT_DELETE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_DELETE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -268,14 +276,16 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService List processInstances = getProcessInstancesByTenant(tenant); if (CollectionUtils.isNotEmpty(processInstances)) { - logger.warn("Delete tenant failed, because there are {} executing process instances using it.", processInstances.size()); + logger.warn("Delete tenant failed, because there are {} executing process instances using it.", + processInstances.size()); throw new ServiceException(Status.DELETE_TENANT_BY_ID_FAIL, processInstances.size()); } List processDefinitions = processDefinitionMapper.queryDefinitionListByTenant(tenant.getId()); if (CollectionUtils.isNotEmpty(processDefinitions)) { - logger.warn("Delete tenant failed, because there are {} process definitions using it.", processDefinitions.size()); + logger.warn("Delete tenant failed, because there are {} process definitions using it.", + processDefinitions.size()); throw new ServiceException(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES, processDefinitions.size()); } @@ -287,7 +297,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService // if resource upload startup if (PropertyUtils.getResUploadStartupState()) { - storageOperate.deleteTenant(tenant.getTenantCode()); + storageOperate.deleteTenant(tenant.getTenantCode()); } int delete = tenantMapper.deleteById(id); @@ -304,7 +314,8 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService } private List getProcessInstancesByTenant(Tenant tenant) { - return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); + return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); } /** @@ -317,7 +328,8 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService public Map queryTenantList(User loginUser) { Map result = new HashMap<>(); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, + loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java index 578668eea6..c5e5da007c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.api.service.impl; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.UdfFuncService; @@ -34,11 +31,8 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; + +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; @@ -46,6 +40,15 @@ import java.util.Date; import java.util.List; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + /** * udf func service impl */ @@ -88,19 +91,21 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic int resourceId) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_CREATE); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_CREATE); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; } // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -119,7 +124,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic return result; } - //save data + // save data UdfFunc udf = new UdfFunc(); Date now = new Date(); udf.setUserId(loginUser.getId()); @@ -165,8 +170,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic @Override public Result queryUdfFuncDetail(User loginUser, int id) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } @@ -206,12 +212,13 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic int resourceId) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId}, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId}, + AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -228,7 +235,8 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -282,14 +290,15 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic @Override public Result queryUdfFuncListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Result result = new Result(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); IPage udfFuncList = getUdfFuncsPage(loginUser, searchVal, pageSize, pageNo); - pageInfo.setTotal((int)udfFuncList.getTotal()); + pageInfo.setTotal((int) udfFuncList.getTotal()); pageInfo.setTotalList(udfFuncList.getRecords()); result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -306,7 +315,8 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic * @return udf function list page */ private IPage getUdfFuncsPage(User loginUser, String searchVal, Integer pageSize, int pageNo) { - Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, loginUser.getId(), logger); + Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, + loginUser.getId(), logger); Page page = new Page<>(pageNo, pageSize); if (udfFuncIds.isEmpty()) { return page; @@ -325,13 +335,15 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic public Result queryUdfFuncList(User loginUser, Integer type) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, loginUser.getId(), logger); - if (udfFuncIds.isEmpty()){ + Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, + loginUser.getId(), logger); + if (udfFuncIds.isEmpty()) { result.setData(Collections.emptyList()); putMsg(result, Status.SUCCESS); return result; @@ -354,8 +366,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic public Result delete(User loginUser, int id) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_DELETE); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_DELETE); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } @@ -375,8 +388,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic @Override public Result verifyUdfFuncByName(User loginUser, String name) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 7ed97daada..47d35bac72 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -56,6 +56,7 @@ import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.apache.dolphinscheduler.service.storage.StorageOperate; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -339,7 +340,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { return result; } if (!isAdmin(loginUser)) { - logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), loginUser.getUserName()); + logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), + loginUser.getUserName()); putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -390,7 +392,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { return result; } if (check(result, !canOperator(loginUser, userId), Status.USER_NO_OPERATION_PERM)) { - logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), loginUser.getUserName()); + logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), + loginUser.getUserName()); return result; } User user = userMapper.selectById(userId); @@ -441,7 +444,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { } if (state == 0 && user.getState() != state && Objects.equals(loginUser.getId(), user.getId())) { - logger.warn("Not allow to disable your own account, userId:{}, userName:{}.", user.getId(), user.getUserName()); + logger.warn("Not allow to disable your own account, userId:{}, userName:{}.", user.getId(), + user.getUserName()); putMsg(result, Status.NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT); return result; } @@ -492,7 +496,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { } // only admin can operate if (!isAdmin(loginUser)) { - logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), loginUser.getUserName()); + logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), + loginUser.getUserName()); putMsg(result, Status.USER_NO_OPERATION_PERM, id); return result; } @@ -508,7 +513,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { if (CollectionUtils.isNotEmpty(projects)) { String projectNames = projects.stream().map(Project::getName).collect(Collectors.joining(",")); putMsg(result, Status.TRANSFORM_PROJECT_OWNERSHIP, projectNames); - logger.warn("Please transfer the project ownership before deleting the user, userId:{}, projects:{}.", id, projectNames); + logger.warn("Please transfer the project ownership before deleting the user, userId:{}, projects:{}.", id, + projectNames); return result; } // delete user @@ -610,7 +616,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { // 3. only project owner can operate if (!this.canOperator(loginUser, project.getUserId())) { - logger.warn("User does not have permission for project, userId:{}, userName:{}, projectCode:{}.", loginUser.getId(), loginUser.getUserName(), projectCode); + logger.warn("User does not have permission for project, userId:{}, userName:{}, projectCode:{}.", + loginUser.getId(), loginUser.getUserName(), projectCode); this.putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -777,7 +784,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { } - logger.info("User is granted permission for resources, userId:{}, resourceIds:{}.", user.getId(), needAuthorizeResIds); + logger.info("User is granted permission for resources, userId:{}, resourceIds:{}.", user.getId(), + needAuthorizeResIds); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java index 791d4acc22..91893f6027 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java @@ -86,7 +86,8 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF putMsg(result, Status.PROJECT_NOT_FOUND, projectCode); return result; } - List workFlowLineageList = workFlowLineageMapper.queryWorkFlowLineageByName(projectCode, workFlowName); + List workFlowLineageList = + workFlowLineageMapper.queryWorkFlowLineageByName(projectCode, workFlowName); result.put(Constants.DATA_LIST, workFlowLineageList); putMsg(result, Status.SUCCESS); return result; @@ -116,23 +117,26 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF long sourceWorkFlowCode, List workFlowLineages, Set workFlowRelations) { - workFlowLineages.add(workFlowLineageMapper.queryWorkFlowLineageByCode(projectCode,sourceWorkFlowCode)); + workFlowLineages.add(workFlowLineageMapper.queryWorkFlowLineageByCode(projectCode, sourceWorkFlowCode)); List downStreamWorkFlowLineages = workFlowLineageMapper.queryDownstreamLineageByProcessDefinitionCode(sourceWorkFlowCode, "DEPENDENT"); workFlowLineages.addAll(downStreamWorkFlowLineages); - downStreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations.add(new WorkFlowRelation(sourceWorkFlowCode, workFlowLineage.getWorkFlowCode()))); + downStreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations + .add(new WorkFlowRelation(sourceWorkFlowCode, workFlowLineage.getWorkFlowCode()))); List upstreamWorkFlowLineages = new ArrayList<>(); getUpstreamLineages(sourceWorkFlowCode, upstreamWorkFlowLineages); workFlowLineages.addAll(upstreamWorkFlowLineages); - upstreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations.add(new WorkFlowRelation(workFlowLineage.getWorkFlowCode(), sourceWorkFlowCode))); + upstreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations + .add(new WorkFlowRelation(workFlowLineage.getWorkFlowCode(), sourceWorkFlowCode))); } private void getUpstreamLineages(long sourceWorkFlowCode, List upstreamWorkFlowLineages) { List workFlowDependentDefinitionList = - workFlowLineageMapper.queryUpstreamDependentParamsByProcessDefinitionCode(sourceWorkFlowCode, "DEPENDENT"); + workFlowLineageMapper.queryUpstreamDependentParamsByProcessDefinitionCode(sourceWorkFlowCode, + "DEPENDENT"); List upstreamProcessDefinitionCodes = new ArrayList<>(); @@ -151,7 +155,8 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF private void getProcessDefinitionCodeByDependentDefinitionList(List dependentDefinitionList, List processDefinitionCodes) { for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) { - for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters().getDependTaskList()) { + for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters() + .getDependTaskList()) { for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) { processDefinitionCodes.add(dependentItem.getDefinitionCode()); @@ -174,8 +179,10 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF Map workFlowLineagesMap = new HashMap<>(); Set workFlowRelations = new HashSet<>(); if (!processLineages.isEmpty()) { - List workFlowLineages = workFlowLineageMapper.queryWorkFlowLineageByLineage(processLineages); - workFlowLineagesMap = workFlowLineages.stream().collect(Collectors.toMap(WorkFlowLineage::getWorkFlowCode, workFlowLineage -> workFlowLineage)); + List workFlowLineages = + workFlowLineageMapper.queryWorkFlowLineageByLineage(processLineages); + workFlowLineagesMap = workFlowLineages.stream() + .collect(Collectors.toMap(WorkFlowLineage::getWorkFlowCode, workFlowLineage -> workFlowLineage)); Map> workFlowMap = new HashMap<>(); for (ProcessLineage processLineage : processLineages) { workFlowMap.compute(processLineage.getProcessDefinitionCode(), (k, v) -> { @@ -186,18 +193,22 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF v.add(new TaskDefinition(processLineage.getPreTaskCode(), processLineage.getPreTaskVersion())); } if (processLineage.getPostTaskCode() > 0) { - v.add(new TaskDefinition(processLineage.getPostTaskCode(), processLineage.getPostTaskVersion())); + v.add(new TaskDefinition(processLineage.getPostTaskCode(), + processLineage.getPostTaskVersion())); } return v; }); } for (Entry> workFlow : workFlowMap.entrySet()) { - Set sourceWorkFlowCodes = querySourceWorkFlowCodes(projectCode, workFlow.getKey(), workFlow.getValue()); + Set sourceWorkFlowCodes = + querySourceWorkFlowCodes(projectCode, workFlow.getKey(), workFlow.getValue()); if (sourceWorkFlowCodes.isEmpty()) { workFlowRelations.add(new WorkFlowRelation(0L, workFlow.getKey())); } else { - workFlowLineagesMap.get(workFlow.getKey()).setSourceWorkFlowCode(StringUtils.join(sourceWorkFlowCodes, Constants.COMMA)); - sourceWorkFlowCodes.forEach(code -> workFlowRelations.add(new WorkFlowRelation(code, workFlow.getKey()))); + workFlowLineagesMap.get(workFlow.getKey()) + .setSourceWorkFlowCode(StringUtils.join(sourceWorkFlowCodes, Constants.COMMA)); + sourceWorkFlowCodes + .forEach(code -> workFlowRelations.add(new WorkFlowRelation(code, workFlow.getKey()))); } } } @@ -209,7 +220,8 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF return result; } - private Set querySourceWorkFlowCodes(long projectCode, long workFlowCode, List taskDefinitionList) { + private Set querySourceWorkFlowCodes(long projectCode, long workFlowCode, + List taskDefinitionList) { Set sourceWorkFlowCodes = new HashSet<>(); if (taskDefinitionList == null || taskDefinitionList.isEmpty()) { return sourceWorkFlowCodes; @@ -218,14 +230,16 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { if (taskDefinitionLog.getProjectCode() == projectCode) { if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) { - DependentParameters dependentParameters = JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); + DependentParameters dependentParameters = + JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); if (dependentParameters != null) { List dependTaskList = dependentParameters.getDependTaskList(); if (!CollectionUtils.isEmpty(dependTaskList)) { for (DependentTaskModel taskModel : dependTaskList) { List dependItemList = taskModel.getDependItemList(); for (DependentItem dependentItem : dependItemList) { - if (dependentItem.getProjectCode() == projectCode && dependentItem.getDefinitionCode() != workFlowCode) { + if (dependentItem.getProjectCode() == projectCode + && dependentItem.getDefinitionCode() != workFlowCode) { sourceWorkFlowCodes.add(dependentItem.getDefinitionCode()); } } @@ -248,14 +262,18 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF */ @Override public Optional taskDepOnTaskMsg(long projectCode, long processDefinitionCode, long taskCode) { - List tasksDep = workFlowLineageMapper.queryTaskDepOnTask(projectCode, processDefinitionCode, taskCode); + List tasksDep = + workFlowLineageMapper.queryTaskDepOnTask(projectCode, processDefinitionCode, taskCode); if (CollectionUtils.isEmpty(tasksDep)) { return Optional.empty(); } - String taskDepStr = tasksDep.stream().map(task -> String.format(Constants.FORMAT_S_S_COLON, task.getProcessDefinitionName(), task.getTaskName())).collect(Collectors.joining(Constants.COMMA)); + String taskDepStr = tasksDep.stream().map( + task -> String.format(Constants.FORMAT_S_S_COLON, task.getProcessDefinitionName(), task.getTaskName())) + .collect(Collectors.joining(Constants.COMMA)); TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCode); - return Optional.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), taskDefinition.getName(), taskDepStr)); + return Optional.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), taskDefinition.getName(), + taskDepStr)); } /** @@ -268,8 +286,10 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF @Override public Set queryTaskDepOnProcess(long projectCode, long processDefinitionCode) { Set taskMainInfos = new HashSet<>(); - List taskDependents = workFlowLineageMapper.queryTaskDependentDepOnProcess(projectCode, processDefinitionCode); - List taskSubProcess = workFlowLineageMapper.queryTaskSubProcessDepOnProcess(projectCode, processDefinitionCode); + List taskDependents = + workFlowLineageMapper.queryTaskDependentDepOnProcess(projectCode, processDefinitionCode); + List taskSubProcess = + workFlowLineageMapper.queryTaskSubProcessDepOnProcess(projectCode, processDefinitionCode); taskMainInfos.addAll(taskDependents); taskMainInfos.addAll(taskSubProcess); return taskMainInfos; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index d95a2017f3..41f4cf8bfa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -322,7 +322,8 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro return result; } List processInstances = processInstanceMapper - .queryByWorkerGroupNameAndStatus(workerGroup.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); + .queryByWorkerGroupNameAndStatus(workerGroup.getName(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); if (CollectionUtils.isNotEmpty(processInstances)) { List processInstanceIds = processInstances.stream().map(ProcessInstance::getId).collect(Collectors.toList()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 33dd23113f..b3e5c95844 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.utils; +import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MAX_LENGTH; +import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MIN_LENGTH; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -31,9 +34,6 @@ import java.util.regex.Pattern; import org.hibernate.validator.internal.constraintvalidators.bv.EmailValidator; -import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MAX_LENGTH; -import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MIN_LENGTH; - /** * check utils */ @@ -67,7 +67,7 @@ public class CheckUtils { if (!emailValidator.isValid(email, null)) { return false; } - //Email is at least a second-level domain name + // Email is at least a second-level domain name int indexDomain = email.lastIndexOf("@"); String domainString = email.substring(indexDomain); return domainString.contains("."); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java index f85a7b111c..94771a4191 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java @@ -36,6 +36,7 @@ import org.springframework.web.multipart.MultipartFile; * file utils */ public class FileUtils { + private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java index b9513b2cc4..ba5735f402 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java @@ -53,7 +53,7 @@ public class PageInfo { private Integer pageNo; public PageInfo() { - + } public PageInfo(Integer currentPage, Integer pageSize) { @@ -128,4 +128,4 @@ public class PageInfo { public void setCurrentPage(Integer currentPage) { this.currentPage = currentPage; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 297fc988b9..d3236786e2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -24,7 +24,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; @@ -134,7 +133,8 @@ public class WorkerGroupControllerTest extends AbstractControllerTest { workerGroup.setId(12); workerGroup.setName("测试"); Mockito.when(workerGroupMapper.selectById(12)).thenReturn(workerGroup); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) .thenReturn(null); Mockito.when(workerGroupMapper.deleteById(12)).thenReturn(1); Mockito.when(processInstanceMapper.updateProcessInstanceByWorkerGroupName("测试", "")).thenReturn(1); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java index 325b499841..bc01e1ba93 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; + import java.util.HashMap; import java.util.Map; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 08b06b720f..7bfaf3ca1e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -284,7 +284,7 @@ public class ProcessInstanceServiceTest { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); projectAuthFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, -1, startTime, endTime); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, -1, startTime, endTime); Assertions.assertEquals(Status.NEGTIVE_SIZE_NUMBER_ERROR, projectAuthFailRes.get(Constants.STATUS)); putMsg(result, Status.SUCCESS, projectCode); @@ -294,7 +294,7 @@ public class ProcessInstanceServiceTest { when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser); Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, - projectCode, size, startTime, endTime); + projectCode, size, startTime, endTime); Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } @@ -314,21 +314,22 @@ public class ProcessInstanceServiceTest { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map startTimeBiggerFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, endTime, startTime); - Assertions.assertEquals(Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, startTimeBiggerFailRes.get(Constants.STATUS)); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, endTime, startTime); + Assertions.assertEquals(Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, + startTimeBiggerFailRes.get(Constants.STATUS)); putMsg(result, Status.SUCCESS, projectCode); when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map dataNullFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, null, endTime); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, null, endTime); Assertions.assertEquals(Status.DATA_IS_NULL, dataNullFailRes.get(Constants.STATUS)); putMsg(result, Status.SUCCESS, projectCode); when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); dataNullFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, startTime, null); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, startTime, null); Assertions.assertEquals(Status.DATA_IS_NULL, dataNullFailRes.get(Constants.STATUS)); } @@ -344,7 +345,7 @@ public class ProcessInstanceServiceTest { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map projectAuthFailRes = - processInstanceService.queryProcessInstanceById(loginUser, projectCode, 1); + processInstanceService.queryProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROJECT_NOT_FOUND, projectAuthFailRes.get(Constants.STATUS)); // project auth success @@ -371,7 +372,7 @@ public class ProcessInstanceServiceTest { Assertions.assertEquals(Status.SUCCESS, workerExistRes.get(Constants.STATUS)); when(processService.findProcessDefinition(processInstance.getProcessDefinitionCode(), - processInstance.getProcessDefinitionVersion())).thenReturn(null);; + processInstance.getProcessDefinitionVersion())).thenReturn(null);; workerExistRes = processInstanceService.queryProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, workerExistRes.get(Constants.STATUS)); } @@ -388,7 +389,7 @@ public class ProcessInstanceServiceTest { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map projectAuthFailRes = - processInstanceService.queryTaskListByProcessId(loginUser, projectCode, 1); + processInstanceService.queryTaskListByProcessId(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROJECT_NOT_FOUND, projectAuthFailRes.get(Constants.STATUS)); // project auth success @@ -638,7 +639,7 @@ public class ProcessInstanceServiceTest { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, INSTANCE_DELETE)).thenReturn(result); Map projectAuthFailRes = - processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); + processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROJECT_NOT_FOUND, projectAuthFailRes.get(Constants.STATUS)); // not sub process @@ -675,7 +676,7 @@ public class ProcessInstanceServiceTest { processDefinition.setProjectCode(projectCode); when(processService.deleteWorkProcessInstanceById(1)).thenReturn(1); Map successRes = - processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); + processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); when(processService.deleteWorkProcessInstanceById(1)).thenReturn(0); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java index b1ae0537a0..b5b99de843 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java @@ -52,6 +52,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 8b38bd101c..3509a288cb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -205,7 +205,8 @@ public class TenantServiceTest { Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.TENANT, null, 0, baseServiceLogger)).thenReturn(true); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) .thenReturn(getInstanceList()); Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList()); Mockito.when(userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java index 5929725cfd..e80233c633 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java @@ -86,7 +86,9 @@ public class WorkerGroupServiceTest { user.setUserType(UserType.ADMIN_USER); WorkerGroup wg2 = getWorkerGroup(2); Mockito.when(workerGroupMapper.selectById(2)).thenReturn(wg2); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList()); + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + .thenReturn(getProcessInstanceList()); Map result = workerGroupService.deleteWorkerGroupById(user, 1); Assertions.assertEquals(Status.DELETE_WORKER_GROUP_NOT_EXIST.getCode(), ((Status) result.get(Constants.STATUS)).getCode()); @@ -96,7 +98,9 @@ public class WorkerGroupServiceTest { // correct WorkerGroup wg3 = getWorkerGroup(3); Mockito.when(workerGroupMapper.selectById(3)).thenReturn(wg3); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(new ArrayList<>()); + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + .thenReturn(new ArrayList<>()); result = workerGroupService.deleteWorkerGroupById(user, 3); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.get(Constants.MSG)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java index 8c9bd2fe74..57467d0a14 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java @@ -17,15 +17,15 @@ package org.apache.dolphinscheduler.common.config; +import java.io.IOException; +import java.util.Properties; + import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; import org.springframework.core.env.PropertiesPropertySource; import org.springframework.core.env.PropertySource; import org.springframework.core.io.support.EncodedResource; import org.springframework.core.io.support.PropertySourceFactory; -import java.io.IOException; -import java.util.Properties; - public class YamlPropertySourceFactory implements PropertySourceFactory { @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java index 0c8ed89fd7..b46a9ed8f7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java @@ -19,5 +19,5 @@ package org.apache.dolphinscheduler.common.enums; public enum AlertEvent { - SERVER_DOWN,TIME_OUT + SERVER_DOWN, TIME_OUT } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java index 14ed47ea2c..de5a87c8ae 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * alert sending(execution) status */ public enum AlertStatus { + /** * 0 waiting executed; 1 execute successfully，2 execute failed */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java index d5713a775a..33c1a360a8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * describe the reason why alert generates */ public enum AlertType { + /** * 0 process instance failure, 1 process instance success, 2 process instance blocked, 3 process instance timeout, 4 fault tolerance warning, * 5 task failure, 6 task success, 7 task timeout, 8 close alert @@ -36,8 +37,7 @@ public enum AlertType { TASK_SUCCESS(6, "task success"), TASK_TIMEOUT(7, "task timeout"), - CLOSE_ALERT(8, "the process instance success, can close the before alert") - ; + CLOSE_ALERT(8, "the process instance success, can close the before alert"); AlertType(int code, String descp) { this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java index 71579a9611..34a15e6d9b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java @@ -19,5 +19,5 @@ package org.apache.dolphinscheduler.common.enums; public enum AlertWarnLevel { - MIDDLE,SERIOUS + MIDDLE, SERIOUS } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java index cb6b6ca0f0..d422cd7de3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java @@ -23,6 +23,7 @@ import java.util.HashMap; * Audit Module type */ public enum AuditResourceType { + // TODO: add other audit resource enums USER_MODULE(0, "USER"), PROJECT_MODULE(1, "PROJECT"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java index f8654f4d1a..49905df3eb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * Authorization type */ public enum AuthorizationType { + /** * 0 RESOURCE_FILE_ID; * 1 RESOURCE_FILE_NAME; @@ -53,14 +54,14 @@ public enum AuthorizationType { ALERT_GROUP(7, "alert group"), ENVIRONMENT(8, "environment"), ACCESS_TOKEN(9, "access token"), - QUEUE(10,"queue"), - DATA_ANALYSIS(11,"data analysis"), - K8S_NAMESPACE(12,"k8s namespace"), - MONITOR(13,"monitor"), - ALERT_PLUGIN_INSTANCE(14,"alert plugin instance"), - TENANT(15,"tenant"), - DATA_QUALITY(16,"data quality"), - TASK_GROUP(17,"task group"), + QUEUE(10, "queue"), + DATA_ANALYSIS(11, "data analysis"), + K8S_NAMESPACE(12, "k8s namespace"), + MONITOR(13, "monitor"), + ALERT_PLUGIN_INSTANCE(14, "alert plugin instance"), + TENANT(15, "tenant"), + DATA_QUALITY(16, "data quality"), + TASK_GROUP(17, "task group"), ; AuthorizationType(int code, String descp) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java index c2d103e5d8..5fc7c5073f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java @@ -24,7 +24,7 @@ public enum BlockingOpportunity { private final String desc; - BlockingOpportunity(String desc){ + BlockingOpportunity(String desc) { this.desc = desc; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java index f845b2083f..b288a9831a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.enums; public enum CacheType { + TENANT("tenant"), USER("user"), QUEUE("queue"), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java index 35b64821af..d87bc39b79 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java @@ -76,7 +76,7 @@ public enum CommandType { static { for (CommandType commandType : CommandType.values()) { - COMMAND_TYPE_MAP.put(commandType.code,commandType); + COMMAND_TYPE_MAP.put(commandType.code, commandType); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java index 68f8e573d2..04c03edf1b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java @@ -23,12 +23,13 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * task node depend type */ public enum ComplementDependentMode { + /** * 0 off mode * 1 run complement data with all dependent process */ - OFF_MODE(0,"off mode"), - ALL_DEPENDENT(1,"all dependent"); + OFF_MODE(0, "off mode"), + ALL_DEPENDENT(1, "all dependent"); ComplementDependentMode(int code, String desc) { this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java index a80f1d38b3..8f690bc8f8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java @@ -26,6 +26,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * condition type */ public enum ConditionType { + /** * 0 none * 1 judge diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java index 83a7069142..80c9cae999 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java @@ -28,6 +28,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * have_alert */ public enum Flag { + /** * 0 no * 1 yes diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java index 278e71e738..91a9c14285 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java @@ -27,5 +27,5 @@ public enum HttpCheckCondition { * 2 body_contains * 3 body_not_contains */ - STATUS_CODE_DEFAULT,STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS -} \ No newline at end of file + STATUS_CODE_DEFAULT, STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java index 509fd7c147..4c225ff2cd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java @@ -29,4 +29,4 @@ public enum HttpMethod { * 4 delete */ GET, POST, HEAD, PUT, DELETE -} \ No newline at end of file +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java index 0f7299ebe5..487ae651e9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java @@ -26,5 +26,5 @@ public enum HttpParametersType { * 1 body; * 2 headers; */ - PARAMETER,BODY,HEADERS + PARAMETER, BODY, HEADERS } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java index b5d5f7d002..08e26b3522 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java @@ -28,7 +28,7 @@ public enum PluginType { ALERT(1, "alert", true), REGISTER(2, "register", false), - TASK(3,"task",true); + TASK(3, "task", true); PluginType(int code, String desc, boolean hasUi) { this.code = code; @@ -53,7 +53,6 @@ public enum PluginType { return hasUi; } - private static HashMap PLUGIN_TYPE_MAP = new HashMap<>(); static { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java index 1ccfb6fe4e..85fd665345 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * define process and task priority */ public enum Priority { + /** * 0 highest priority * 1 higher priority diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java index 87952becd4..e11375aa5f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java @@ -41,12 +41,12 @@ public enum ReleaseState { private final String descp; public static ReleaseState getEnum(int value) { - for (ReleaseState e:ReleaseState.values()) { + for (ReleaseState e : ReleaseState.values()) { if (e.ordinal() == value) { return e; } } - //For values out of enum scope + // For values out of enum scope return null; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java index 9d231cce00..f55905e745 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java @@ -22,7 +22,8 @@ import com.baomidou.mybatisplus.annotation.EnumValue; /** * complement data run mode */ -public enum RunMode { +public enum RunMode { + /** * 0 serial run * 1 parallel run diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java index 731911bc3b..b37a3338d1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java @@ -19,5 +19,5 @@ package org.apache.dolphinscheduler.common.enums; public enum SlotCheckState { - PASS,INJECT,CHANGE + PASS, INJECT, CHANGE } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java index 2f69f060cc..17b244c745 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java @@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; -public enum SqoopJobType { +public enum SqoopJobType { + CUSTOM(0, "CUSTOM"), TEMPLATE(1, "TEMPLATE"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java index 0301947564..7bc8b9d410 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * task node depend type */ public enum TaskDependType { + /** * 0 run current tasks only * 1 run current tasks and previous tasks diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java index 4694cec26d..c2c7f3a165 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * task execute type */ public enum TaskExecuteType { + /** * 0 batch * 1 stream diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java index faa9d0ceab..a57ad3fe47 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java @@ -29,7 +29,6 @@ public enum TaskRecordStatus { * 1 failure * 2 exception */ - SUCCESS,FAILURE,EXCEPTION - + SUCCESS, FAILURE, EXCEPTION } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java index f5ef952b02..9088575812 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * timeout flag */ public enum TimeoutFlag { + /** * 0 close * 1 open diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java index 211061681b..9c4266bf7a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java @@ -23,13 +23,14 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * UDF type */ public enum UdfType { + /** * 0 hive; 1 spark */ - HIVE(0, "hive"), - SPARK(1, "spark"); + HIVE(0, "hive"), + SPARK(1, "spark"); - UdfType(int code, String descp){ + UdfType(int code, String descp) { this.code = code; this.descp = descp; } @@ -46,14 +47,13 @@ public enum UdfType { return descp; } - public static UdfType of(int type){ - for(UdfType ut : values()){ - if(ut.getCode() == type){ + public static UdfType of(int type) { + for (UdfType ut : values()) { + if (ut.getCode() == type) { return ut; } } throw new IllegalArgumentException("invalid type : " + type); } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java index ebab5da742..effc2cfc1e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * user type */ public enum UserType { + /** * 0 admin user; 1 general user */ @@ -46,4 +47,3 @@ public enum UserType { return descp; } } - diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java index 65f31cc911..2ba69e944e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java @@ -29,6 +29,7 @@ import com.google.common.base.Functions; * types for whether to send warning when process ends; */ public enum WarningType { + /** * 0 do not send warning; * 1 send if process success; @@ -58,7 +59,7 @@ public enum WarningType { } private static final Map WARNING_TYPE_MAP = - Arrays.stream(WarningType.values()).collect(toMap(WarningType::getDescp, Functions.identity())); + Arrays.stream(WarningType.values()).collect(toMap(WarningType::getDescp, Functions.identity())); public static WarningType of(String descp) { if (WARNING_TYPE_MAP.containsKey(descp)) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java index 65e2b8d946..4fced36110 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java @@ -17,13 +17,15 @@ package org.apache.dolphinscheduler.common.enums; -import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.NonNull; - import java.util.HashMap; import java.util.Map; +import lombok.NonNull; + +import com.baomidou.mybatisplus.annotation.EnumValue; + public enum WorkflowExecutionStatus { + // This class is split from ExecutionStatus #11339. // In order to compatible with the old value, the code is not consecutive SUBMITTED_SUCCESS(0, "submit success"), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java index 3aaf7980d0..b5ab837a84 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java @@ -37,7 +37,8 @@ public class StorageOperateNoConfiguredException extends RuntimeException { super(cause); } - public StorageOperateNoConfiguredException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public StorageOperateNoConfiguredException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java index afa780163a..fb5a564cf6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java @@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory; * EdgeInfo: edge description information */ public class DAG { + private static final Logger logger = LoggerFactory.getLogger(DAG.class); private final ReadWriteLock lock = new ReentrantReadWriteLock(); @@ -466,8 +467,7 @@ public class DAG { } /* - * After scanning, there is no node with 0 degree of entry, - * indicating that there is a ring, and return directly + * After scanning, there is no node with 0 degree of entry, indicating that there is a ring, and return directly */ if (zeroIndegreeNodeQueue.isEmpty()) { return new AbstractMap.SimpleEntry<>(false, topoResultList); @@ -501,13 +501,12 @@ public class DAG { @Override public String toString() { return "DAG{" - + "nodesMap=" - + nodesMap - + ", edgesMap=" - + edgesMap - + ", reverseEdgesMap=" - + reverseEdgesMap - + '}'; + + "nodesMap=" + + nodesMap + + ", edgesMap=" + + edgesMap + + ", reverseEdgesMap=" + + reverseEdgesMap + + '}'; } } - diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java index 557f7ec86b..e703ce0f17 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.common.model; -import lombok.extern.slf4j.Slf4j; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; +import lombok.extern.slf4j.Slf4j; + @Slf4j public abstract class BaseHeartBeatTask extends BaseDaemonThread { @@ -48,7 +49,8 @@ public abstract class BaseHeartBeatTask extends BaseDaemonThread { while (runningFlag) { try { if (!ServerLifeCycleManager.isRunning()) { - log.info("The current server status is {}, will not write heartBeatInfo into registry", ServerLifeCycleManager.getServerStatus()); + log.info("The current server status is {}, will not write heartBeatInfo into registry", + ServerLifeCycleManager.getServerStatus()); continue; } T heartBeat = getHeartBeat(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java index 6ca51a4c29..e9552f507d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java @@ -27,6 +27,7 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @AllArgsConstructor public class MasterHeartBeat implements HeartBeat { + private long startupTime; private long reportTime; private double cpuUsage; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java index 020357578e..f439184982 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java @@ -37,7 +37,7 @@ public class PreviousTaskNode { public PreviousTaskNode() { } - + public PreviousTaskNode(long code, String name, int version) { this.code = code; this.name = name; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java index 743979eed7..873a69bcaf 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.common.model; -import lombok.Data; - import java.util.Date; +import lombok.Data; + @Data public class Server { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java index 7ee8322601..6abedd39a3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java @@ -60,7 +60,7 @@ public class TaskNodeRelation { if (!(o instanceof TaskNodeRelation)) { return false; } - TaskNodeRelation relation = (TaskNodeRelation)o; + TaskNodeRelation relation = (TaskNodeRelation) o; return (relation.getStartNode().equals(this.startNode) && relation.getEndNode().equals(this.endNode)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java index 7980e9bbed..4bc5c044bc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java @@ -44,5 +44,4 @@ public class WorkerHeartBeat implements HeartBeat { private int workerWaitingTaskCount; // worker waiting task count private int workerExecThreadCount; // worker thread pool thread count - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java index ab259f4181..11786fd5a3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.HttpParametersType; import java.util.Objects; public class HttpProperty { + /** * key */ @@ -115,9 +116,9 @@ public class HttpProperty { @Override public String toString() { return "HttpProperty{" - + "prop='" + prop + '\'' - + ", httpParametersType=" + httpParametersType - + ", value='" + value + '\'' - + '}'; + + "prop='" + prop + '\'' + + ", httpParametersType=" + httpParametersType + + ", value='" + value + '\'' + + '}'; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index c2f63cce14..913f4500e6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -39,10 +39,8 @@ import org.slf4j.LoggerFactory; * time-intervals. */ public abstract class AbstractShell { - - private static final Logger logger = LoggerFactory.getLogger(AbstractShell.class); - + private static final Logger logger = LoggerFactory.getLogger(AbstractShell.class); /** * Time after which the executing script would be timedout @@ -79,11 +77,11 @@ public abstract class AbstractShell { * If or not script finished executing */ private AtomicBoolean completed; - + public AbstractShell() { this(0L); } - + /** * @param interval the minimum duration to wait before re-executing the * command. @@ -145,7 +143,7 @@ public abstract class AbstractShell { if (timeOutInterval > 0) { timeOutTimer = new Timer(); timeoutTimerTask = new ShellTimeoutTimerTask(this); - //One time scheduling. + // One time scheduling. timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); } final BufferedReader errReader = @@ -159,6 +157,7 @@ public abstract class AbstractShell { // read error and input streams as this would free up the buffers // free the error stream buffer Thread errThread = new Thread() { + @Override public void run() { try { @@ -174,6 +173,7 @@ public abstract class AbstractShell { } }; Thread inThread = new Thread() { + @Override public void run() { try { @@ -200,9 +200,9 @@ public abstract class AbstractShell { } catch (InterruptedException ie) { logger.warn("Interrupted while reading the error and in stream", ie); } - completed.compareAndSet(false,true); - //the timeout thread handling - //taken care in finally block + completed.compareAndSet(false, true); + // the timeout thread handling + // taken care in finally block if (exitCode != 0 || errMsg.length() > 0) { throw new ExitCodeException(exitCode, errMsg.toString()); } @@ -237,7 +237,7 @@ public abstract class AbstractShell { * @return an array containing the command name and its parameters */ protected abstract String[] getExecString(); - + /** * Parse the execution result * @param lines lines @@ -285,9 +285,9 @@ public abstract class AbstractShell { try { p.exitValue(); } catch (Exception e) { - //Process has not terminated. - //So check if it has completed - //if not just destroy it. + // Process has not terminated. + // So check if it has completed + // if not just destroy it. if (p != null && !shell.completed.get()) { shell.setTimedOut(); p.destroy(); @@ -300,6 +300,7 @@ public abstract class AbstractShell { * This is an IOException with exit code added. */ public static class ExitCodeException extends IOException { + int exitCode; public ExitCodeException(int exitCode, String message) { @@ -317,6 +318,7 @@ public abstract class AbstractShell { * */ public static class ProcessContainer extends ConcurrentHashMap { + private static final ProcessContainer container = new ProcessContainer(); private ProcessContainer() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java index ac1eae590a..1e12d0ed5f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java @@ -44,8 +44,8 @@ public class ShellExecutor extends AbstractShell { } public ShellExecutor(String[] execString, File dir, - Map env) { - this(execString, dir, env,0L); + Map env) { + this(execString, dir, env, 0L); } /** @@ -63,7 +63,7 @@ public class ShellExecutor extends AbstractShell { * If 0, the command will not be timed out. */ public ShellExecutor(String[] execString, File dir, - Map env, long timeout) { + Map env, long timeout) { command = execString.clone(); if (dir != null) { setWorkingDirectory(dir); @@ -113,8 +113,7 @@ public class ShellExecutor extends AbstractShell { * @return the output of the executed command. * @throws IOException errors */ - public static String execCommand(Map env, String... cmd) - throws IOException { + public static String execCommand(Map env, String... cmd) throws IOException { return execCommand(env, cmd, 0L); } @@ -138,7 +137,7 @@ public class ShellExecutor extends AbstractShell { int nRead; String line = ""; while ((nRead = lines.read(buf, 0, buf.length)) > 0) { - line = new String(buf,0,nRead); + line = new String(buf, 0, nRead); output.append(line); } } @@ -151,7 +150,6 @@ public class ShellExecutor extends AbstractShell { return (output == null) ? "" : output.toString(); } - /** * Returns the commands of this instance. * Arguments with spaces in are presented with quotes round; other diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index f4f2a17bc7..6eca0e783f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -21,13 +21,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import lombok.experimental.UtilityClass; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.experimental.UtilityClass; - @UtilityClass public class ThreadUtils { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java index 0ac3b5db01..ae24894c33 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java @@ -75,7 +75,8 @@ public class CollectionUtils { * @param T * @return removes certain attributes of each object in the list */ - public static List> getListByExclusion(List originList, Set exclusionSet) { + public static List> getListByExclusion(List originList, + Set exclusionSet) { List> instanceList = new ArrayList<>(); if (originList == null) { return instanceList; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 08890e86fb..be503dd5b0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -22,8 +22,6 @@ import org.apache.dolphinscheduler.common.thread.ThreadLocalContext; import org.apache.commons.lang3.StringUtils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -33,6 +31,9 @@ import java.util.Calendar; import java.util.Date; import java.util.TimeZone; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +49,7 @@ public final class DateUtils { private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final DateTimeFormatter YYYY_MM_DD_HH_MM_SS = - DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS); + DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS); private DateUtils() { throw new UnsupportedOperationException("Construct DateUtils"); @@ -123,7 +124,8 @@ public final class DateUtils { public static String format(Date date, DateTimeFormatter dateTimeFormatter, String timezone) { LocalDateTime localDateTime = - StringUtils.isEmpty(timezone) ? date2LocalDateTime(date) : date2LocalDateTime(date, ZoneId.of(timezone)); + StringUtils.isEmpty(timezone) ? date2LocalDateTime(date) + : date2LocalDateTime(date, ZoneId.of(timezone)); return format(localDateTime, dateTimeFormatter); } @@ -243,8 +245,8 @@ public final class DateUtils { Date d = stringToDate(date); if (d == null) { throw new IllegalArgumentException(String.format( - "data: %s should be a validate data string - yyyy-MM-dd HH:mm:ss ", - date)); + "data: %s should be a validate data string - yyyy-MM-dd HH:mm:ss ", + date)); } return ZonedDateTime.ofInstant(d.toInstant(), ZoneId.systemDefault()); } @@ -612,9 +614,9 @@ public final class DateUtils { } String dateToString = dateToString(date, sourceTimezoneId); LocalDateTime localDateTime = - LocalDateTime.parse(dateToString, DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS)); + LocalDateTime.parse(dateToString, DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS)); ZonedDateTime zonedDateTime = - ZonedDateTime.of(localDateTime, TimeZone.getTimeZone(targetTimezoneId).toZoneId()); + ZonedDateTime.of(localDateTime, TimeZone.getTimeZone(targetTimezoneId).toZoneId()); return Date.from(zonedDateTime.toInstant()); } @@ -632,6 +634,7 @@ public final class DateUtils { * Time unit representing one thousandth of a second */ public static class MILLISECONDS { + public static long toDays(long d) { return d / (C6 / C2); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index 23e4b74b75..beb64167c6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -57,7 +57,8 @@ public class FileUtils { * @return download file name */ public static String getDownloadFilename(String filename) { - String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); + String fileName = + String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); File file = new File(fileName); if (!file.getParentFile().exists()) { @@ -94,7 +95,8 @@ public class FileUtils { * @param taskInstanceId task instance id * @return directory of process execution */ - public static String getProcessExecDir(long projectCode, long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) { + public static String getProcessExecDir(long projectCode, long processDefineCode, int processDefineVersion, + int processInstanceId, int taskInstanceId) { String fileName = String.format("%s/exec/process/%d/%s/%d/%d", DATA_BASEDIR, projectCode, processDefineCode + "_" + processDefineVersion, processInstanceId, taskInstanceId); File file = new File(fileName); @@ -119,7 +121,7 @@ public class FileUtils { * @throws IOException errors */ public static void createWorkDirIfAbsent(String execLocalPath) throws IOException { - //if work dir exists, first delete + // if work dir exists, first delete File execLocalPathFile = new File(execLocalPath); if (execLocalPathFile.exists()) { @@ -134,7 +136,7 @@ public class FileUtils { } } - //create work dir + // create work dir org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile); String mkdirLog = "create dir success " + execLocalPath; logger.info(mkdirLog); @@ -230,7 +232,7 @@ public class FileUtils { * @param filename String type of filename * @return whether file path could be traversal or not */ - public static boolean directoryTraversal(String filename){ + public static boolean directoryTraversal(String filename) { if (filename.contains(FOLDER_SEPARATOR)) { return true; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 100d60b131..83557865ff 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -67,6 +67,7 @@ public class HttpUtils { } private static class HttpClientInstance { + private static final CloseableHttpClient httpClient = getHttpClientBuilder().build(); } @@ -85,6 +86,7 @@ public class HttpUtils { private static Registry socketFactoryRegistry; private static X509TrustManager xtm = new X509TrustManager() { + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { } @@ -102,7 +104,7 @@ public class HttpUtils { static { try { ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); - ctx.init(null, new TrustManager[] {xtm}, null); + ctx.init(null, new TrustManager[]{xtm}, null); } catch (NoSuchAlgorithmException e) { logger.error("SSLContext init with NoSuchAlgorithmException", e); } catch (KeyManagementException e) { @@ -111,14 +113,14 @@ public class HttpUtils { socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); /** set timeout、request time、socket timeout */ requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .setExpectContinueEnabled(Boolean.TRUE) - .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST, AuthSchemes.SPNEGO)) - .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC, AuthSchemes.SPNEGO)) - .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) - .build(); + .setExpectContinueEnabled(Boolean.TRUE) + .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST, AuthSchemes.SPNEGO)) + .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC, AuthSchemes.SPNEGO)) + .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) + .build(); socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); + .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); cm.setDefaultMaxPerRoute(60); cm.setMaxTotal(100); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java index c6d2508af9..9262dbd9b9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java @@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory; * kerberos http client */ public class KerberosHttpClient { + public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class); private String principal; @@ -75,6 +76,7 @@ public class KerberosHttpClient { HttpClientBuilder builder = HttpUtils.getHttpClientBuilder(); BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() { + @Override public Principal getUserPrincipal() { return null; @@ -92,6 +94,7 @@ public class KerberosHttpClient { public String get(final String url, final String userId) { logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url); Configuration config = new Configuration() { + @SuppressWarnings("serial") @Override public AppConfigurationEntry[] getAppConfigurationEntry(String name) { @@ -105,9 +108,9 @@ public class KerberosHttpClient { options.put("doNotPrompt", "true"); options.put("isInitiator", "true"); options.put("debug", "true"); - return new AppConfigurationEntry[] { - new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)}; + return new AppConfigurationEntry[]{ + new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)}; } }; Set princ = new HashSet<>(1); @@ -140,9 +143,9 @@ public class KerberosHttpClient { String responseContent; KerberosHttpClient kerberosHttpClient = new KerberosHttpClient( - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)); return responseContent; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java index 138ff25f16..30c03b0cb8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java @@ -30,6 +30,7 @@ import com.github.rholder.retry.StopStrategies; import com.github.rholder.retry.WaitStrategies; public class RetryerUtils { + private static Retryer defaultRetryerResultCheck; private static Retryer defaultRetryerResultNoCheck; @@ -92,7 +93,8 @@ public class RetryerUtils { * @throws ExecutionException the execution exception * @throws RetryException the retry exception */ - public static Boolean retryCall(final Callable callable, boolean checkResult) throws ExecutionException, RetryException { + public static Boolean retryCall(final Callable callable, + boolean checkResult) throws ExecutionException, RetryException { return getDefaultRetryer(checkResult).call(callable); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java index ab92df8f9b..26be151941 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.common.utils.placeholder; +import static org.apache.commons.lang3.time.DateUtils.addDays; import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_DATE; import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME; import static org.apache.dolphinscheduler.common.utils.DateUtils.format; -import static org.apache.commons.lang3.time.DateUtils.addDays; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -34,6 +33,7 @@ import java.util.Map; * business time utils */ public class BusinessTimeUtils { + private BusinessTimeUtils() { throw new IllegalStateException("BusinessTimeUtils class"); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java index 96b438031d..7026b2a87d 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common; import org.apache.commons.lang3.SystemUtils; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java index e49d8d9b29..6d7d7fe799 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java @@ -17,14 +17,13 @@ package org.apache.dolphinscheduler.common.exception; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ExceptionTest { @Test - public void testException(){ + public void testException() { final String message = "Test"; RuntimeException time = new RuntimeException(message); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java index deff8a2ea5..5b3958dbf5 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java @@ -16,6 +16,10 @@ */ package org.apache.dolphinscheduler.common.graph; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -23,341 +27,322 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - public class DAGTest { - private DAG graph; - private static final Logger logger = LoggerFactory.getLogger(DAGTest.class); - - @BeforeEach - public void setup() { - graph = new DAG<>(); - } - - @AfterEach - public void tearDown() { - clear(); - } - - private void clear() { - graph = null; - graph = new DAG<>(); - Assertions.assertEquals(graph.getNodesCount(), 0); - } + private DAG graph; + private static final Logger logger = LoggerFactory.getLogger(DAGTest.class); - - private void makeGraph() { - clear(); - - // 1->2 - // 2->5 - // 3->5 - // 4->6 - // 5->6 - // 6->7 - - for (int i = 1; i <= 7; ++i) { - graph.addNode(i, "v(" + i + ")"); + @BeforeEach + public void setup() { + graph = new DAG<>(); } - // construction side - Assertions.assertTrue(graph.addEdge(1, 2)); - - Assertions.assertTrue(graph.addEdge(2, 5)); - - Assertions.assertTrue(graph.addEdge(3, 5)); - - Assertions.assertTrue(graph.addEdge(4, 6)); - - Assertions.assertTrue(graph.addEdge(5, 6)); - - Assertions.assertTrue(graph.addEdge(6, 7)); - - Assertions.assertEquals(graph.getNodesCount(), 7); - Assertions.assertEquals(graph.getEdgesCount(), 6); - - } - - - /** - * add node - */ - @Test - public void testAddNode() { - clear(); - - graph.addNode(1, "v(1)"); - graph.addNode(2, null); - graph.addNode(5, "v(5)"); - - Assertions.assertEquals(graph.getNodesCount(), 3); - - Assertions.assertEquals(graph.getNode(1), "v(1)"); - Assertions.assertTrue(graph.containsNode(1)); + @AfterEach + public void tearDown() { + clear(); + } - Assertions.assertFalse(graph.containsNode(10)); - } + private void clear() { + graph = null; + graph = new DAG<>(); + Assertions.assertEquals(graph.getNodesCount(), 0); + } - /** - * add edge - */ - @Test - public void testAddEdge() { - clear(); + private void makeGraph() { + clear(); - Assertions.assertFalse(graph.addEdge(1, 2, "edge(1 -> 2)", false)); + // 1->2 + // 2->5 + // 3->5 + // 4->6 + // 5->6 + // 6->7 - graph.addNode(1, "v(1)"); + for (int i = 1; i <= 7; ++i) { + graph.addNode(i, "v(" + i + ")"); + } - Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)",true)); + // construction side + Assertions.assertTrue(graph.addEdge(1, 2)); - graph.addNode(2, "v(2)"); + Assertions.assertTrue(graph.addEdge(2, 5)); - Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)",true)); + Assertions.assertTrue(graph.addEdge(3, 5)); - Assertions.assertFalse(graph.containsEdge(1, 3)); + Assertions.assertTrue(graph.addEdge(4, 6)); - Assertions.assertTrue(graph.containsEdge(1, 2)); - Assertions.assertEquals(graph.getEdgesCount(), 1); + Assertions.assertTrue(graph.addEdge(5, 6)); - int node = 3; - graph.addNode(node, "v(3)"); - Assertions.assertFalse(graph.addEdge(node, node)); + Assertions.assertTrue(graph.addEdge(6, 7)); - } + Assertions.assertEquals(graph.getNodesCount(), 7); + Assertions.assertEquals(graph.getEdgesCount(), 6); + } - /** - * add subsequent node - */ - @Test - public void testSubsequentNodes() { - makeGraph(); + /** + * add node + */ + @Test + public void testAddNode() { + clear(); - Assertions.assertEquals(graph.getSubsequentNodes(1).size(), 1); + graph.addNode(1, "v(1)"); + graph.addNode(2, null); + graph.addNode(5, "v(5)"); - } + Assertions.assertEquals(graph.getNodesCount(), 3); + Assertions.assertEquals(graph.getNode(1), "v(1)"); + Assertions.assertTrue(graph.containsNode(1)); - /** - * test indegree - */ - @Test - public void testIndegree() { - makeGraph(); + Assertions.assertFalse(graph.containsNode(10)); + } - Assertions.assertEquals(graph.getIndegree(1), 0); - Assertions.assertEquals(graph.getIndegree(2), 1); - Assertions.assertEquals(graph.getIndegree(3), 0); - Assertions.assertEquals(graph.getIndegree(4), 0); - } + /** + * add edge + */ + @Test + public void testAddEdge() { + clear(); + Assertions.assertFalse(graph.addEdge(1, 2, "edge(1 -> 2)", false)); - /** - * test begin node - */ - @Test - public void testBeginNode() { - makeGraph(); + graph.addNode(1, "v(1)"); - Assertions.assertEquals(graph.getBeginNode().size(), 3); + Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)", true)); - Assertions.assertTrue(graph.getBeginNode().contains(1)); - Assertions.assertTrue(graph.getBeginNode().contains(3)); - Assertions.assertTrue(graph.getBeginNode().contains(4)); - } + graph.addNode(2, "v(2)"); + Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)", true)); - /** - * test end node - */ - @Test - public void testEndNode() { - makeGraph(); + Assertions.assertFalse(graph.containsEdge(1, 3)); - Assertions.assertEquals(graph.getEndNode().size(), 1); + Assertions.assertTrue(graph.containsEdge(1, 2)); + Assertions.assertEquals(graph.getEdgesCount(), 1); - Assertions.assertTrue(graph.getEndNode().contains(7)); - } + int node = 3; + graph.addNode(node, "v(3)"); + Assertions.assertFalse(graph.addEdge(node, node)); + } - /** - * test cycle - */ - @Test - public void testCycle() { - clear(); + /** + * add subsequent node + */ + @Test + public void testSubsequentNodes() { + makeGraph(); + Assertions.assertEquals(graph.getSubsequentNodes(1).size(), 1); - for (int i = 1; i <= 5; ++i) { - graph.addNode(i, "v(" + i + ")"); } - // construction side - try { - graph.addEdge(1, 2); - graph.addEdge(2, 3); - graph.addEdge(3, 4); - - Assertions.assertFalse(graph.hasCycle()); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + /** + * test indegree + */ + @Test + public void testIndegree() { + makeGraph(); + + Assertions.assertEquals(graph.getIndegree(1), 0); + Assertions.assertEquals(graph.getIndegree(2), 1); + Assertions.assertEquals(graph.getIndegree(3), 0); + Assertions.assertEquals(graph.getIndegree(4), 0); } + /** + * test begin node + */ + @Test + public void testBeginNode() { + makeGraph(); - try { - boolean addResult = graph.addEdge(4, 1); + Assertions.assertEquals(graph.getBeginNode().size(), 3); - if(!addResult){ - Assertions.assertTrue(true); - } - - graph.addEdge(5, 1); - - Assertions.assertFalse(graph.hasCycle()); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + Assertions.assertTrue(graph.getBeginNode().contains(1)); + Assertions.assertTrue(graph.getBeginNode().contains(3)); + Assertions.assertTrue(graph.getBeginNode().contains(4)); } - clear(); + /** + * test end node + */ + @Test + public void testEndNode() { + makeGraph(); + + Assertions.assertEquals(graph.getEndNode().size(), 1); - // construction node - for (int i = 1; i <= 5; ++i) { - graph.addNode(i, "v(" + i +")"); + Assertions.assertTrue(graph.getEndNode().contains(7)); } - // construction side, 1->2, 2->3, 3->4 - try { - graph.addEdge(1, 2); - graph.addEdge(2, 3); - graph.addEdge(3, 4); - graph.addEdge(4, 5); - graph.addEdge(5, 2);//会失败，添加不进去，所以下一步无环 - - Assertions.assertFalse(graph.hasCycle()); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + /** + * test cycle + */ + @Test + public void testCycle() { + clear(); + + for (int i = 1; i <= 5; ++i) { + graph.addNode(i, "v(" + i + ")"); + } + + // construction side + try { + graph.addEdge(1, 2); + graph.addEdge(2, 3); + graph.addEdge(3, 4); + + Assertions.assertFalse(graph.hasCycle()); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } + + try { + boolean addResult = graph.addEdge(4, 1); + + if (!addResult) { + Assertions.assertTrue(true); + } + + graph.addEdge(5, 1); + + Assertions.assertFalse(graph.hasCycle()); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } + + clear(); + + // construction node + for (int i = 1; i <= 5; ++i) { + graph.addNode(i, "v(" + i + ")"); + } + + // construction side, 1->2, 2->3, 3->4 + try { + graph.addEdge(1, 2); + graph.addEdge(2, 3); + graph.addEdge(3, 4); + graph.addEdge(4, 5); + graph.addEdge(5, 2);// 会失败，添加不进去，所以下一步无环 + + Assertions.assertFalse(graph.hasCycle()); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } } - } - - - @Test - public void testTopologicalSort(){ - makeGraph(); - - try { - // topological result is : 1 3 4 2 5 6 7 - List topoList = new ArrayList<>(); - topoList.add(1); - topoList.add(3); - topoList.add(4); - topoList.add(2); - topoList.add(5); - topoList.add(6); - topoList.add(7); - - Assertions.assertEquals(graph.topologicalSort(),topoList); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + + @Test + public void testTopologicalSort() { + makeGraph(); + + try { + // topological result is : 1 3 4 2 5 6 7 + List topoList = new ArrayList<>(); + topoList.add(1); + topoList.add(3); + topoList.add(4); + topoList.add(2); + topoList.add(5); + topoList.add(6); + topoList.add(7); + + Assertions.assertEquals(graph.topologicalSort(), topoList); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } } - } + @Test + public void testTopologicalSort2() { + clear(); - @Test - public void testTopologicalSort2() { - clear(); + graph.addEdge(1, 2, null, true); + graph.addEdge(2, 3, null, true); + graph.addEdge(3, 4, null, true); + graph.addEdge(4, 5, null, true); + graph.addEdge(5, 1, null, false); // The loop will fail to add - graph.addEdge(1, 2, null, true); - graph.addEdge(2, 3, null, true); - graph.addEdge(3, 4, null, true); - graph.addEdge(4, 5, null, true); - graph.addEdge(5, 1, null, false); //The loop will fail to add + try { + List topoList = new ArrayList<>();// topological result is : 1 2 3 4 5 + topoList.add(1); + topoList.add(2); + topoList.add(3); + topoList.add(4); + topoList.add(5); - try { - List topoList = new ArrayList<>();// topological result is : 1 2 3 4 5 - topoList.add(1); - topoList.add(2); - topoList.add(3); - topoList.add(4); - topoList.add(5); + Assertions.assertEquals(graph.topologicalSort(), topoList); - Assertions.assertEquals(graph.topologicalSort(),topoList); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); } - } - + @Test + public void testTopologicalSort3() throws Exception { + clear(); - @Test - public void testTopologicalSort3() throws Exception { - clear(); + // 1->2 + // 1->3 + // 2->5 + // 3->4 + // 4->6 + // 5->6 + // 6->7 + // 6->8 - // 1->2 - // 1->3 - // 2->5 - // 3->4 - // 4->6 - // 5->6 - // 6->7 - // 6->8 + for (int i = 1; i <= 8; ++i) { + graph.addNode(i, "v(" + i + ")"); + } - for (int i = 1; i <= 8; ++i) { - graph.addNode(i, "v(" + i + ")"); - } - - // construction node - Assertions.assertTrue(graph.addEdge(1, 2)); - - Assertions.assertTrue(graph.addEdge(1, 3)); + // construction node + Assertions.assertTrue(graph.addEdge(1, 2)); - Assertions.assertTrue(graph.addEdge(2, 5)); - Assertions.assertTrue(graph.addEdge(3, 4)); + Assertions.assertTrue(graph.addEdge(1, 3)); - Assertions.assertTrue(graph.addEdge(4, 6)); + Assertions.assertTrue(graph.addEdge(2, 5)); + Assertions.assertTrue(graph.addEdge(3, 4)); - Assertions.assertTrue(graph.addEdge(5, 6)); + Assertions.assertTrue(graph.addEdge(4, 6)); - Assertions.assertTrue(graph.addEdge(6, 7)); - Assertions.assertTrue(graph.addEdge(6, 8)); + Assertions.assertTrue(graph.addEdge(5, 6)); + Assertions.assertTrue(graph.addEdge(6, 7)); + Assertions.assertTrue(graph.addEdge(6, 8)); + Assertions.assertEquals(graph.getNodesCount(), 8); + logger.info(Arrays.toString(graph.topologicalSort().toArray())); - Assertions.assertEquals(graph.getNodesCount(), 8); + List expectedList = new ArrayList<>(); - logger.info(Arrays.toString(graph.topologicalSort().toArray())); + for (int i = 1; i <= 8; ++i) { + expectedList.add(i); - List expectedList = new ArrayList<>(); + logger.info(i + " subsequentNodes : " + graph.getSubsequentNodes(i)); + } + logger.info(6 + " previousNodesb: " + graph.getPreviousNodes(6)); + Assertions.assertEquals(5, graph.getSubsequentNodes(2).toArray()[0]); - for (int i = 1; i <= 8; ++i) { - expectedList.add(i); - - logger.info(i + " subsequentNodes : " + graph.getSubsequentNodes(i)); } - logger.info(6 + " previousNodesb: " + graph.getPreviousNodes(6)); - Assertions.assertEquals(5, graph.getSubsequentNodes(2).toArray()[0]); - - } - - @Test - public void testTopologicalSort4() { - clear(); - try { - graph.topologicalSort(); - } catch (Exception e) { - Assertions.assertTrue(e.getMessage().contains("serious error: graph has cycle")); + + @Test + public void testTopologicalSort4() { + clear(); + try { + graph.topologicalSort(); + } catch (Exception e) { + Assertions.assertTrue(e.getMessage().contains("serious error: graph has cycle")); + } } - } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index 7525d1c76e..561f2a73f2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -16,15 +16,17 @@ */ package org.apache.dolphinscheduler.common.os; -import org.apache.commons.lang3.SystemUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; + +import org.apache.commons.lang3.SystemUtils; + +import java.util.List; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - /** * OSUtilsTest */ diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java index e484b61f8d..3871646c95 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.common.utils; +import java.util.HashSet; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.HashSet; - public class CodeGenerateUtilsTest { + @Test public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { HashSet existsCode = new HashSet<>(); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java index 36ffe1134a..dc07c1cdaa 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java @@ -18,12 +18,7 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.thread.ThreadLocalContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import javax.management.timer.Timer; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.ZoneId; @@ -31,6 +26,13 @@ import java.time.ZonedDateTime; import java.util.Date; import java.util.TimeZone; +import javax.management.timer.Timer; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + public class DateUtilsTest { @BeforeEach @@ -226,7 +228,8 @@ public class DateUtilsTest { Assertions.assertEquals(DateUtils.dateToString(date), DateUtils.dateToString(defaultTimeZoneDate)); Date targetTimeZoneDate = DateUtils.transformTimezoneDate(date, TimeZone.getDefault().getID(), "Asia/Shanghai"); - Assertions.assertEquals(DateUtils.dateToString(date, TimeZone.getDefault().getID()), DateUtils.dateToString(targetTimeZoneDate, "Asia/Shanghai")); + Assertions.assertEquals(DateUtils.dateToString(date, TimeZone.getDefault().getID()), + DateUtils.dateToString(targetTimeZoneDate, "Asia/Shanghai")); } @Test @@ -286,7 +289,7 @@ public class DateUtilsTest { @Test public void testTimeStampToDate() { long timeStamp = 1664456400000L; - SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); sdf.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); String sd = sdf.format(new Date(timeStamp)); Assertions.assertEquals("2022-09-29 21:00:00", sd); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java index 28db71b5a7..9f9710efd0 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.api.Test; */ public class EncryptionUtilsTest { - @Test public void testGetMd5() { Assertions.assertEquals(EncryptionUtils.getMd5(null), EncryptionUtils.getMd5("")); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index c5e6e3354b..7fbd77d8bc 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -17,7 +17,13 @@ package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; + import org.apache.dolphinscheduler.common.Constants; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -25,11 +31,6 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.FileInputStream; -import java.io.FileNotFoundException; - -import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; - @ExtendWith(MockitoExtension.class) public class FileUtilsTest { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java index c4ff8bfc13..1275784cdc 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java @@ -19,19 +19,24 @@ package org.apache.dolphinscheduler.common.utils; import java.io.IOException; import java.io.OutputStream; -import java.net.BindException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import junit.extensions.TestSetup; import junit.framework.Test; -import org.mortbay.jetty.*; + +import org.mortbay.jetty.HttpConnection; +import org.mortbay.jetty.Request; +import org.mortbay.jetty.Server; import org.mortbay.jetty.handler.AbstractHandler; import org.mortbay.jetty.handler.ContextHandler; import org.mortbay.util.ByteArrayISO8859Writer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; public class LocalJettyHttpServer extends TestSetup { + protected static Server server; private static Logger logger = LoggerFactory.getLogger(LocalJettyHttpServer.class); private Integer serverPort = 0; @@ -44,13 +49,14 @@ public class LocalJettyHttpServer extends TestSetup { super(suite); } - protected void setUp() throws Exception { server = new Server(serverPort); ContextHandler context = new ContextHandler("/test.json"); context.setHandler(new AbstractHandler() { + @Override - public void handle(String s, HttpServletRequest request, HttpServletResponse response, int i) throws IOException { + public void handle(String s, HttpServletRequest request, HttpServletResponse response, + int i) throws IOException { ByteArrayISO8859Writer writer = new ByteArrayISO8859Writer(); writer.write("{\"name\":\"Github\"}"); writer.flush(); @@ -58,7 +64,8 @@ public class LocalJettyHttpServer extends TestSetup { OutputStream out = response.getOutputStream(); writer.writeTo(out); out.flush(); - Request baseRequest = request instanceof Request ? (Request) request : HttpConnection.getCurrentConnection().getRequest(); + Request baseRequest = request instanceof Request ? (Request) request + : HttpConnection.getCurrentConnection().getRequest(); baseRequest.setHandled(true); } }); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java index f03066ef49..ac1e75bab7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java @@ -17,19 +17,23 @@ package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.databind.node.ObjectNode; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; import org.apache.dolphinscheduler.common.Constants; + import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + public class LocalServerHttpUtilsTest extends TestCase { public static final Logger logger = LoggerFactory.getLogger(LocalServerHttpUtilsTest.class); @@ -58,8 +62,8 @@ public class LocalServerHttpUtilsTest extends TestCase { HttpGet httpget = new HttpGet("http://localhost:" + server.getServerPort() + "/test.json"); /** set timeout、request time、socket timeout */ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) - .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build(); + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) + .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build(); httpget.setConfig(requestConfig); String responseContent = null; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java index 20625f29c2..8a17395784 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java @@ -45,7 +45,8 @@ public class NetUtilsTest { when(address.getCanonicalHostName()) .thenReturn("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local"); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); - Assertions.assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); + Assertions.assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", + NetUtils.getHost(address)); address = mock(InetAddress.class); when(address.getCanonicalHostName()) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java index b4c6426372..49e1dd35ed 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java index 8fcbdea5d0..a1a157337b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java @@ -16,12 +16,13 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.github.rholder.retry.RetryException; -import com.github.rholder.retry.Retryer; +import java.util.concurrent.ExecutionException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.concurrent.ExecutionException; +import com.github.rholder.retry.RetryException; +import com.github.rholder.retry.Retryer; public class RetryerUtilsTest { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java index cb983ae450..7b1b96b56f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -16,15 +16,16 @@ */ package org.apache.dolphinscheduler.common.utils; +import java.io.StringReader; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.io.StringReader; - public class ScriptRunnerTest { + @Test public void testRunScript() { - //connection is null + // connection is null Exception exception = null; ScriptRunner s = new ScriptRunner(null, true, true); try { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java index d818eef21a..04bf267a51 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java @@ -16,24 +16,24 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class StringTest { @Test - public void stringCompareTest(){ + public void stringCompareTest() { - for(int j = 0; j < 5; j++) { + for (int j = 0; j < 5; j++) { long start = System.currentTimeMillis(); int size = 10000; List taskList = new ArrayList<>(size); - //init + // init for (int i = 0; i < size; i++) { taskList.add(String.format("%d_%010d_%010d", 1, i, i + 1)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index dfdf4521a6..470dffb588 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao; -import org.apache.commons.lang3.math.NumberUtils; import org.apache.dolphinscheduler.common.enums.AlertEvent; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; @@ -39,6 +38,7 @@ import org.apache.dolphinscheduler.dao.mapper.AlertSendStatusMapper; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.math.NumberUtils; import java.time.LocalDateTime; import java.time.ZoneId; @@ -67,7 +67,7 @@ public class AlertDao { * logger of AlertDao */ private static final Logger logger = LoggerFactory.getLogger(AlertDao.class); - + private static final int QUERY_ALERT_THRESHOLD = 100; @Value("${alert.alarm-suppression.crash:60}") diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java index 02f9dbd75d..109a204f11 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java @@ -25,6 +25,7 @@ import java.util.Date; * cycle dependency */ public class CycleDependency { + /** * process define id */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java index b6754a3b67..f180627ac6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java @@ -39,7 +39,8 @@ public class DagData { */ private List taskDefinitionList; - public DagData(ProcessDefinition processDefinition, List processTaskRelationList, List taskDefinitionList) { + public DagData(ProcessDefinition processDefinition, List processTaskRelationList, + List taskDefinitionList) { this.processDefinition = processDefinition; this.processTaskRelationList = processTaskRelationList; this.taskDefinitionList = taskDefinitionList; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java index a80a7e06d3..441f1be852 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.*; import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.Date; import lombok.Data; +import com.baomidou.mybatisplus.annotation.FieldFill; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java index 87bb3d4234..17f60f928c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java @@ -89,13 +89,13 @@ public class DependentProcessDefinition { cycleEnum = CycleEnum.DAY; break; case "hour": - cycleEnum = CycleEnum.HOUR; + cycleEnum = CycleEnum.HOUR; break; case "week": - cycleEnum = CycleEnum.WEEK; + cycleEnum = CycleEnum.WEEK; break; case "month": - cycleEnum = CycleEnum.MONTH; + cycleEnum = CycleEnum.MONTH; break; default: break; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java index cd7f24b081..7ec52868b2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java @@ -35,7 +35,7 @@ public class DqExecuteResultAlertContent implements Serializable { * process define name */ @JsonProperty("processDefinitionName") - private String processDefinitionName; + private String processDefinitionName; /** * process_instance_id */ @@ -141,8 +141,9 @@ public class DqExecuteResultAlertContent implements Serializable { } public static class Builder { + private long processDefinitionId; - private String processDefinitionName; + private String processDefinitionName; private long processInstanceId; private String processInstanceName; private long taskInstanceId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index d83a3dc624..efacc1c92b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.TableField; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; @@ -29,6 +28,7 @@ import java.util.Date; import lombok.Data; import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java index ffd45e49fa..4530402040 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @Data @NoArgsConstructor diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java index 3f2f858964..e9e3c88642 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java @@ -17,14 +17,15 @@ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + @NoArgsConstructor @Getter @Setter diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java index 22dab95234..31f89d1348 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java @@ -17,18 +17,24 @@ package org.apache.dolphinscheduler.dao.entity; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.dolphinscheduler.common.enums.AlertEvent; +import org.apache.dolphinscheduler.common.enums.AlertWarnLevel; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + +import java.io.Serializable; +import java.util.Date; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.common.enums.*; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import java.io.Serializable; -import java.util.Date; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @AllArgsConstructor diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java index 6876c060f8..3a0bf32947 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java @@ -26,6 +26,7 @@ import lombok.Data; @Data @AllArgsConstructor public class ResponseTaskLog { + private int lineNum; private String message; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java index 211863f73f..6ab298e296 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java @@ -49,6 +49,7 @@ public class ServerAlertContent { } public static class Builder { + private String type; private String host; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java index 39bf6143dd..7929b3de6a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java @@ -16,12 +16,12 @@ */ package org.apache.dolphinscheduler.dao.entity; +import java.util.Date; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import java.util.Date; - /** * session */ @@ -31,7 +31,7 @@ public class Session { /** * id */ - @TableId(value="id", type=IdType.INPUT) + @TableId(value = "id", type = IdType.INPUT) private String id; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java index fbd2f64a59..5d2f1c8e08 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java @@ -17,18 +17,19 @@ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + +import java.io.Serializable; +import java.util.Date; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; -import java.util.Date; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @Data @Builder diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java index 68b63141c5..7eb4cb3820 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.dao.entity; import java.util.Date; public class WorkFlowLineage { + private long workFlowCode; private String workFlowName; private String workFlowPublishStatus; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java index f89992ec13..72dbf8790d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.dao.entity; import java.util.Objects; public class WorkFlowRelation { + private long sourceWorkFlowCode; private long targetWorkFlowCode; @@ -57,7 +58,7 @@ public class WorkFlowRelation { } WorkFlowRelation that = (WorkFlowRelation) o; return sourceWorkFlowCode == that.sourceWorkFlowCode - && targetWorkFlowCode == that.targetWorkFlowCode; + && targetWorkFlowCode == that.targetWorkFlowCode; } @Override diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java index 7d45c66940..16d67b07aa 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java @@ -78,14 +78,13 @@ public class ZookeeperRecord { */ private Date date; - /** * is normal or not, 1:normal */ private int state; - - public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, float minLatency, float avgLatency, float maxLatency, int nodeCount, int state,Date date) { + public ZookeeperRecord(String hostname, int connections, int watches, long sent, long received, String mode, + float minLatency, float avgLatency, float maxLatency, int nodeCount, int state, Date date) { this.hostname = hostname; this.connections = connections; this.watches = watches; @@ -100,7 +99,6 @@ public class ZookeeperRecord { this.date = date; } - public String getHostname() { return hostname; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java index b6fabb66c1..8165768932 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java @@ -32,7 +32,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ public interface AccessTokenMapper extends BaseMapper { - /** * access token page * @@ -43,8 +42,7 @@ public interface AccessTokenMapper extends BaseMapper { */ IPage selectAccessTokenPage(Page page, @Param("userName") String userName, - @Param("userId") int userId - ); + @Param("userId") int userId); /** * Query access token for specified user @@ -68,5 +66,6 @@ public interface AccessTokenMapper extends BaseMapper { * @param accessTokensIds * @return access token for specified user */ - List listAuthorizedAccessToken(@Param("userId") int userId, @Param("accessTokensIds")List accessTokensIds); + List listAuthorizedAccessToken(@Param("userId") int userId, + @Param("accessTokensIds") List accessTokensIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java index ceabc5b49b..712b67e5f2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java @@ -19,10 +19,8 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.AlertGroup; -import org.apache.dolphinscheduler.dao.entity.User; import org.apache.ibatis.annotations.Param; -import java.util.ArrayList; import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; @@ -34,7 +32,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; */ public interface AlertGroupMapper extends BaseMapper { - /** * alertgroup page * @param page page @@ -44,7 +41,6 @@ public interface AlertGroupMapper extends BaseMapper { IPage queryAlertGroupPage(Page page, @Param("groupName") String groupName); - /** * query by group name * @param groupName groupName @@ -91,7 +87,8 @@ public interface AlertGroupMapper extends BaseMapper { * @param alertGroupsIds * @return */ - List listAuthorizedAlertGroupList (@Param("userId") int userId, @Param("alertGroupsIds")List alertGroupsIds); + List listAuthorizedAlertGroupList(@Param("userId") int userId, + @Param("alertGroupsIds") List alertGroupsIds); /** * queryAlertGroupPageByIds @@ -100,5 +97,6 @@ public interface AlertGroupMapper extends BaseMapper { * @param searchVal * @return */ - IPage queryAlertGroupPageByIds(Page page, @Param("ids") List ids, @Param("searchVal") String searchVal); + IPage queryAlertGroupPageByIds(Page page, @Param("ids") List ids, + @Param("searchVal") String searchVal); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java index bfd86aa23e..7d77b3e429 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java @@ -30,6 +30,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; * auditlog mapper interface */ public interface AuditLogMapper extends BaseMapper { + IPage queryAuditLog(IPage page, @Param("resourceType") int[] resourceArray, @Param("operationType") int[] operationType, diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index de0ea222a1..877f7442e2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -40,9 +40,9 @@ public interface CommandMapper extends BaseMapper { * @return CommandCount list */ List countCommandState( - @Param("startTime") Date startTime, - @Param("endTime") Date endTime, - @Param("projectCodeArray") Long[] projectCodeArray); + @Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("projectCodeArray") Long[] projectCodeArray); /** * query command page @@ -50,10 +50,11 @@ public interface CommandMapper extends BaseMapper { */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); - /** * query command page by slot * @return command list */ - List queryCommandPageBySlot(@Param("limit") int limit, @Param("offset") int offset, @Param("masterCount") int masterCount, @Param("thisMasterSlot") int thisMasterSlot); + List queryCommandPageBySlot(@Param("limit") int limit, @Param("offset") int offset, + @Param("masterCount") int masterCount, + @Param("thisMasterSlot") int thisMasterSlot); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java index b70c5f801c..288e65089b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.ibatis.annotations.Param; -import java.util.ArrayList; import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * datasource mapper interface @@ -39,7 +38,8 @@ public interface DataSourceMapper extends BaseMapper { * @param type type * @return datasource list */ - List queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type, @Param("testFlag") int testFlag); + List queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type, + @Param("testFlag") int testFlag); /** * datasource page @@ -59,7 +59,6 @@ public interface DataSourceMapper extends BaseMapper { */ List queryDataSourceByName(@Param("name") String name); - /** * query authed datasource * @param userId userId @@ -81,7 +80,6 @@ public interface DataSourceMapper extends BaseMapper { */ List listAllDataSourceByType(@Param("type") Integer type); - /** * list authorized UDF function * @@ -90,7 +88,8 @@ public interface DataSourceMapper extends BaseMapper { * @param T * @return UDF function list */ - List listAuthorizedDataSource(@Param("userId") int userId, @Param("dataSourceIds") T[] dataSourceIds); + List listAuthorizedDataSource(@Param("userId") int userId, + @Param("dataSourceIds") T[] dataSourceIds); /** * query datasource by name and user id @@ -108,7 +107,9 @@ public interface DataSourceMapper extends BaseMapper { * @param searchVal * @return */ - IPage selectPagingByIds(Page dataSourcePage, @Param("dataSourceIds") List dataSourceIds, @Param("name") String name); + IPage selectPagingByIds(Page dataSourcePage, + @Param("dataSourceIds") List dataSourceIds, + @Param("name") String name); /** * clearBindTestId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java index df51ebe33e..209d65cfc9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java @@ -28,7 +28,6 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; */ public interface DataSourceUserMapper extends BaseMapper { - /** * delete datasource by userId * @param userId userId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java index fbc1cb7a9a..7688259c4f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.ibatis.annotations.Param; import java.util.List; -import java.util.Set; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * environment mapper interface @@ -78,5 +77,6 @@ public interface EnvironmentMapper extends BaseMapper { * @param searchVal * @return */ - IPage queryEnvironmentListPagingByIds(Page page, @Param("ids")List ids, @Param("searchName")String searchVal); + IPage queryEnvironmentListPagingByIds(Page page, @Param("ids") List ids, + @Param("searchName") String searchVal); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java index acd9ce79cb..5c04e217e1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java @@ -40,7 +40,7 @@ public interface ErrorCommandMapper extends BaseMapper { * @return CommandCount list */ List countCommandState( - @Param("startTime") Date startTime, - @Param("endTime") Date endTime, - @Param("projectCodeArray") Long[] projectCodeArray); + @Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("projectCodeArray") Long[] projectCodeArray); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java index 0c102ce263..8ea39dd7d8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.dolphinscheduler.dao.entity.FavTask; + import org.apache.ibatis.annotations.Param; import java.util.Set; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * fav mapper interface */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java index f89e7579e4..ce0f2f87a4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java @@ -1,83 +1,84 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.mapper; - -import org.apache.dolphinscheduler.dao.entity.K8sNamespace; - -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; - -/** - * namespace interface - */ -public interface K8sNamespaceMapper extends BaseMapper { - /** - * k8s namespace page - * - * @param page page - * @param searchVal searchVal - * @return k8s namespace IPage - */ - IPage queryK8sNamespacePaging(IPage page, - @Param("searchVal") String searchVal); - - /** - * check the target namespace exist - * - * @param namespace namespace - * @param clusterCode clusterCode - * @return true if exist else return null - */ - Boolean existNamespace(@Param("namespace") String namespace, @Param("clusterCode") Long clusterCode); - - /** - * query namespace except userId - * - * @param userId userId - * @return namespace list - */ - List queryNamespaceExceptUserId(@Param("userId") int userId); - - /** - * query authed namespace list by userId - * - * @param userId userId - * @return namespace list - */ - List queryAuthedNamespaceListByUserId(@Param("userId") int userId); - - /** - * query namespace can use - * - * @param userId userId - * @return namespace list - */ - List queryNamespaceAvailable(@Param("userId") Integer userId); - - /** - * check the target namespace - * - * @param namespaceCode namespaceCode - * @return true if exist else return null - */ - K8sNamespace queryByNamespaceCode(@Param("clusterCode") Long namespaceCode); -} \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.K8sNamespace; + +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + +/** + * namespace interface + */ +public interface K8sNamespaceMapper extends BaseMapper { + + /** + * k8s namespace page + * + * @param page page + * @param searchVal searchVal + * @return k8s namespace IPage + */ + IPage queryK8sNamespacePaging(IPage page, + @Param("searchVal") String searchVal); + + /** + * check the target namespace exist + * + * @param namespace namespace + * @param clusterCode clusterCode + * @return true if exist else return null + */ + Boolean existNamespace(@Param("namespace") String namespace, @Param("clusterCode") Long clusterCode); + + /** + * query namespace except userId + * + * @param userId userId + * @return namespace list + */ + List queryNamespaceExceptUserId(@Param("userId") int userId); + + /** + * query authed namespace list by userId + * + * @param userId userId + * @return namespace list + */ + List queryAuthedNamespaceListByUserId(@Param("userId") int userId); + + /** + * query namespace can use + * + * @param userId userId + * @return namespace list + */ + List queryNamespaceAvailable(@Param("userId") Integer userId); + + /** + * check the target namespace + * + * @param namespaceCode namespaceCode + * @return true if exist else return null + */ + K8sNamespace queryByNamespaceCode(@Param("clusterCode") Long namespaceCode); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java index bd9297ec74..ba2d6cb9e3 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java @@ -53,7 +53,8 @@ public interface ProcessDefinitionLogMapper extends BaseMapper queryByDefinitionName(@Param("projectCode") long projectCode, @Param("name") String name); + List queryByDefinitionName(@Param("projectCode") long projectCode, + @Param("name") String name); /** * query process definition log list @@ -81,7 +82,9 @@ public interface ProcessDefinitionLogMapper extends BaseMapper queryProcessDefinitionVersionsPaging(Page page, @Param("code") long code, @Param("projectCode") long projectCode); + IPage queryProcessDefinitionVersionsPaging(Page page, + @Param("code") long code, + @Param("projectCode") long projectCode); /** * delete the certain process definition version by process definition id and version number diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java index 0397d37154..fd3440e2d5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; + import org.apache.ibatis.annotations.Param; import java.util.List; -import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; import com.baomidou.mybatisplus.core.mapper.BaseMapper; /** @@ -38,7 +39,6 @@ public interface ProcessInstanceMapMapper extends BaseMapper ProcessInstanceMap queryByParentId(@Param("parentProcessId") int parentProcessId, @Param("parentTaskId") int parentTaskId); - /** * query by sub process id * @param subProcessId subProcessId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java index 3a3c314b36..59c635f711 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java @@ -32,6 +32,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; * project mapper interface */ public interface ProjectMapper extends BaseMapper { + /** * query project detail by code * @param projectCode projectCode @@ -128,7 +129,6 @@ public interface ProjectMapper extends BaseMapper { */ List queryAllProject(@Param("userId") int userId); - /** * list authorized Projects * @param userId @@ -136,7 +136,7 @@ public interface ProjectMapper extends BaseMapper { * @param * @return */ - List listAuthorizedProjects(@Param("userId") int userId, @Param("projectsIds")List projectsIds); + List listAuthorizedProjects(@Param("userId") int userId, @Param("projectsIds") List projectsIds); /** * query all project for dependent node diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java index 0b523e8619..f37ed0f5a1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.Queue; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * queue mapper interface */ @@ -34,7 +36,7 @@ public interface QueueMapper extends BaseMapper { * @param searchVal searchVal * @return queue IPage */ - IPage queryQueuePaging(IPage page, @Param("ids")List ids, + IPage queryQueuePaging(IPage page, @Param("ids") List ids, @Param("searchVal") String searchVal); /** @@ -44,7 +46,7 @@ public interface QueueMapper extends BaseMapper { * @return queue list */ List queryAllQueueList(@Param("queue") String queue, - @Param("queueName") String queueName); + @Param("queueName") String queueName); /** * check the target queue exist diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index 268e7bafb3..45c70cb723 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -97,14 +97,14 @@ public interface ResourceMapper extends BaseMapper { * @param T * @return resource list */ - List listAuthorizedResourceById(@Param("userId") int userId,@Param("resIds")T[] resIds); + List listAuthorizedResourceById(@Param("userId") int userId, @Param("resIds") T[] resIds); /** * delete resource by id array * @param resIds resource id array * @return delete num */ - int deleteIds(@Param("resIds")Integer[] resIds); + int deleteIds(@Param("resIds") Integer[] resIds); /** * list children @@ -119,14 +119,14 @@ public interface ResourceMapper extends BaseMapper { * @param type resource type * @return resource */ - List queryResource(@Param("fullName") String fullName,@Param("type") int type); + List queryResource(@Param("fullName") String fullName, @Param("type") int type); /** * list resource by id array * @param resIds resource id array * @return resource list */ - List listResourceByIds(@Param("resIds")Integer[] resIds); + List listResourceByIds(@Param("resIds") Integer[] resIds); /** * update resource @@ -143,8 +143,8 @@ public interface ResourceMapper extends BaseMapper { * @return true if exist else return null */ Boolean existResourceByUser(@Param("fullName") String fullName, - @Param("userId") int userId, - @Param("type") int type); + @Param("userId") int userId, + @Param("type") int type); /** * check resource exist diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java index dbaca256cb..6fc36ea425 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java @@ -55,6 +55,6 @@ public interface ResourceUserMapper extends BaseMapper { * @return delete result */ int deleteResourceUserArray(@Param("userId") int userId, - @Param("resIds") Integer[] resIds); + @Param("resIds") Integer[] resIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java index a72a680710..8f6a7597ee 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java @@ -17,11 +17,13 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.Session; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * session mapper interface */ @@ -40,6 +42,6 @@ public interface SessionMapper extends BaseMapper { * @param ip ip * @return session */ - Session queryByUserIdAndIp(@Param("userId") int userId,@Param("ip") String ip); + Session queryByUserIdAndIp(@Param("userId") int userId, @Param("ip") String ip); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java index 851d8dac51..98cd1175e1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java @@ -94,5 +94,6 @@ public interface TaskDefinitionLogMapper extends BaseMapper { * @param code process definition code * @return the paging task definition version list */ - IPage queryTaskDefinitionVersionsPaging(Page page, @Param("code") long code, @Param("projectCode") long projectCode); + IPage queryTaskDefinitionVersionsPaging(Page page, @Param("code") long code, + @Param("projectCode") long projectCode); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java index 05c99194ce..819e70dcfb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java @@ -21,12 +21,12 @@ import org.apache.dolphinscheduler.dao.entity.TaskGroup; import org.apache.ibatis.annotations.Param; +import java.util.List; + import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import java.util.List; - /** * the Dao interfaces of task group * @@ -79,9 +79,10 @@ public interface TaskGroupMapper extends BaseMapper { */ int selectAvailableCountById(@Param("groupId") int groupId); - int selectCountByIdStatus(@Param("id") int id,@Param("status") int status); + int selectCountByIdStatus(@Param("id") int id, @Param("status") int status); - IPage queryTaskGroupPagingByProjectCode(Page page, @Param("ids") List ids, @Param("projectCode") Long projectCode); + IPage queryTaskGroupPagingByProjectCode(Page page, @Param("ids") List ids, + @Param("projectCode") Long projectCode); /** * listAuthorizedResource diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java index 38f5ca6016..9826b3e6d5 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java @@ -44,8 +44,7 @@ public interface TaskGroupQueueMapper extends BaseMapper { * @return task group queue list */ IPage queryTaskGroupQueuePaging(IPage page, - @Param("groupId") int groupId - ); + @Param("groupId") int groupId); TaskGroupQueue queryByTaskId(@Param("taskId") int taskId); @@ -77,7 +76,8 @@ public interface TaskGroupQueueMapper extends BaseMapper { /** * Query the {@link TaskGroupQueue}, who's priority > the given priority */ - List queryHighPriorityTasks(@Param("groupId") int groupId, @Param("priority") int priority, @Param("status") int status); + List queryHighPriorityTasks(@Param("groupId") int groupId, @Param("priority") int priority, + @Param("status") int status); TaskGroupQueue queryTheHighestPriorityTasks(@Param("groupId") int groupId, @Param("status") int status, @Param("forceStart") int forceStart, @Param("inQueue") int inQueue); @@ -86,14 +86,17 @@ public interface TaskGroupQueueMapper extends BaseMapper { void updateForceStart(@Param("queueId") int queueId, @Param("forceStart") int forceStart); - int updateInQueueLimit1(@Param("oldValue") int oldValue, @Param("newValue") int newValue - , @Param("groupId") int id, @Param("status") int status); + int updateInQueueLimit1(@Param("oldValue") int oldValue, @Param("newValue") int newValue, @Param("groupId") int id, + @Param("status") int status); int updateInQueueCAS(@Param("oldValue") int oldValue, @Param("newValue") int newValue, @Param("id") int id); void modifyPriority(@Param("queueId") int queueId, @Param("priority") int priority); - IPage queryTaskGroupQueueByTaskGroupIdPaging(Page page, @Param("taskName")String taskName - ,@Param("processName") String processName,@Param("status") Integer status,@Param("groupId") int groupId - ,@Param("projects") List projects); + IPage queryTaskGroupQueueByTaskGroupIdPaging(Page page, + @Param("taskName") String taskName, + @Param("processName") String processName, + @Param("status") Integer status, + @Param("groupId") int groupId, + @Param("projects") List projects); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java index cdc26ee4e6..7b90ba24af 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java @@ -17,18 +17,20 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import org.apache.ibatis.annotations.Param; import java.util.Date; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * task instance mapper interface */ @@ -55,7 +57,7 @@ public interface TaskInstanceMapper extends BaseMapper { @Param("taskCode") Long taskCode); List queryByProcessInstanceIdsAndTaskCodes(@Param("processInstanceIds") List processInstanceIds, - @Param("taskCodes") List taskCodes); + @Param("taskCodes") List taskCodes); Integer countTask(@Param("projectCodes") Long[] projectCodes, @Param("taskIds") int[] taskIds); @@ -100,20 +102,19 @@ public interface TaskInstanceMapper extends BaseMapper { @Param("host") String host, @Param("taskExecuteType") TaskExecuteType taskExecuteType, @Param("startTime") Date startTime, - @Param("endTime") Date endTime - ); + @Param("endTime") Date endTime); IPage queryStreamTaskInstanceListPaging(IPage page, - @Param("projectCode") Long projectCode, - @Param("processDefinitionName") String processDefinitionName, - @Param("searchVal") String searchVal, - @Param("taskName") String taskName, - @Param("executorId") int executorId, - @Param("states") int[] statusArray, - @Param("host") String host, - @Param("taskExecuteType") TaskExecuteType taskExecuteType, - @Param("startTime") Date startTime, - @Param("endTime") Date endTime); + @Param("projectCode") Long projectCode, + @Param("processDefinitionName") String processDefinitionName, + @Param("searchVal") String searchVal, + @Param("taskName") String taskName, + @Param("executorId") int executorId, + @Param("states") int[] statusArray, + @Param("host") String host, + @Param("taskExecuteType") TaskExecuteType taskExecuteType, + @Param("startTime") Date startTime, + @Param("endTime") Date endTime); List loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId, @Param("status") int status); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java index 7da6741183..623c1f8a8d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java @@ -17,20 +17,19 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.ibatis.annotations.Param; +import java.util.List; + import org.springframework.cache.annotation.CacheConfig; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; - -import java.util.ArrayList; -import java.util.List; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * tenant mapper interface @@ -74,7 +73,7 @@ public interface TenantMapper extends BaseMapper { * @param searchVal searchVal * @return tenant IPage */ - IPage queryTenantPaging(IPage page,@Param("ids") List ids, + IPage queryTenantPaging(IPage page, @Param("ids") List ids, @Param("searchVal") String searchVal); /** @@ -92,7 +91,8 @@ public interface TenantMapper extends BaseMapper { * @param searchVal * @return */ - IPage queryTenantPagingByIds(Page page, @Param("ids")List ids, @Param("searchVal")String searchVal); + IPage queryTenantPagingByIds(Page page, @Param("ids") List ids, + @Param("searchVal") String searchVal); /** * queryAll diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java index 6c7026dadf..2a5bc6a417 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java @@ -17,9 +17,11 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.UDFUser; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; + import org.apache.ibatis.annotations.Param; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * udf user realtion mapper interface */ @@ -40,4 +42,3 @@ public interface UDFUserMapper extends BaseMapper { int deleteByUdfFuncId(@Param("udfFuncId") int udfFuncId); } - diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 340c3e9ce1..5573aedd8d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.UdfFunc; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * udf function mapper interface */ @@ -84,7 +86,7 @@ public interface UdfFuncMapper extends BaseMapper { * @param udfIds UDF function id array * @return UDF function list */ - List listAuthorizedUdfFunc (@Param("userId") int userId,@Param("udfIds")T[] udfIds); + List listAuthorizedUdfFunc(@Param("userId") int userId, @Param("udfIds") T[] udfIds); /** * list UDF by resource id @@ -98,7 +100,7 @@ public interface UdfFuncMapper extends BaseMapper { * @param resourceIds resource id array * @return UDF function list */ - List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + List listAuthorizedUdfByResourceId(@Param("userId") int userId, @Param("resourceIds") int[] resourceIds); /** * batch update udf func diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java index a58299806f..249f0b9f4a 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java @@ -80,7 +80,6 @@ public interface UserMapper extends BaseMapper { */ User queryUserByNamePassword(@Param("userName") String userName, @Param("password") String password); - /** * user page * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java index 951f12a14a..b47731c591 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java @@ -35,7 +35,8 @@ public interface WorkFlowLineageMapper { * @param workFlowName workFlowName * @return WorkFlowLineage list */ - List queryWorkFlowLineageByName(@Param("projectCode") long projectCode, @Param("workFlowName") String workFlowName); + List queryWorkFlowLineageByName(@Param("projectCode") long projectCode, + @Param("workFlowName") String workFlowName); /** * queryWorkFlowLineageByCode @@ -44,7 +45,8 @@ public interface WorkFlowLineageMapper { * @param workFlowCode workFlowCode * @return WorkFlowLineage */ - WorkFlowLineage queryWorkFlowLineageByCode(@Param("projectCode") long projectCode, @Param("workFlowCode") long workFlowCode); + WorkFlowLineage queryWorkFlowLineageByCode(@Param("projectCode") long projectCode, + @Param("workFlowCode") long workFlowCode); /** * queryWorkFlowLineageByProcessDefinitionCodes @@ -95,7 +97,6 @@ public interface WorkFlowLineageMapper { List queryDownstreamLineageByProcessDefinitionCode(@Param("code") long code, @Param("taskType") String taskType); - /** * query upstream work flow dependent task params by process definition code * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java index 46c7610fde..f2cb65877f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java @@ -41,9 +41,11 @@ public class JsonSplitDao { * @param processDefinitionLogs processDefinitionLogs */ public void executeJsonSplitProcessDefinition(Connection conn, List processDefinitionLogs) { - String updateSql = "UPDATE t_ds_process_definition SET global_params=?,timeout=?,tenant_id=?,locations=?,update_time=? where id=?"; - String insertLogSql = "insert into t_ds_process_definition_log (code,name,version,description,project_code,release_state,user_id," - + "global_params,flag,locations,timeout,tenant_id,operator,operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String updateSql = + "UPDATE t_ds_process_definition SET global_params=?,timeout=?,tenant_id=?,locations=?,update_time=? where id=?"; + String insertLogSql = + "insert into t_ds_process_definition_log (code,name,version,description,project_code,release_state,user_id," + + "global_params,flag,locations,timeout,tenant_id,operator,operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { PreparedStatement processUpdate = conn.prepareStatement(updateSql); PreparedStatement insertLog = conn.prepareStatement(insertLogSql); @@ -101,12 +103,15 @@ public class JsonSplitDao { * @param conn jdbc connection * @param processTaskRelationLogs processTaskRelationLogs */ - public void executeJsonSplitProcessTaskRelation(Connection conn, List processTaskRelationLogs) { - String insertSql = "insert into t_ds_process_task_relation (project_code,process_definition_code,process_definition_version,pre_task_code,pre_task_version," - + "post_task_code,post_task_version,condition_type,condition_params,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?)"; - String insertLogSql = "insert into t_ds_process_task_relation_log (project_code,process_definition_code,process_definition_version,pre_task_code," - + "pre_task_version,post_task_code,post_task_version,condition_type,condition_params,operator,operate_time,create_time,update_time) " - + "values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + public void executeJsonSplitProcessTaskRelation(Connection conn, + List processTaskRelationLogs) { + String insertSql = + "insert into t_ds_process_task_relation (project_code,process_definition_code,process_definition_version,pre_task_code,pre_task_version," + + "post_task_code,post_task_version,condition_type,condition_params,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?)"; + String insertLogSql = + "insert into t_ds_process_task_relation_log (project_code,process_definition_code,process_definition_version,pre_task_code," + + "pre_task_version,post_task_code,post_task_version,condition_type,condition_params,operator,operate_time,create_time,update_time) " + + "values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { PreparedStatement insert = conn.prepareStatement(insertSql); PreparedStatement insertLog = conn.prepareStatement(insertLogSql); @@ -167,12 +172,14 @@ public class JsonSplitDao { * @param taskDefinitionLogs taskDefinitionLogs */ public void executeJsonSplitTaskDefinition(Connection conn, List taskDefinitionLogs) { - String insertSql = "insert into t_ds_task_definition (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," - + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids," - + "create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - String insertLogSql = "insert into t_ds_task_definition_log (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," - + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids,operator," - + "operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String insertSql = + "insert into t_ds_task_definition (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," + + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids," + + "create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String insertLogSql = + "insert into t_ds_task_definition_log (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," + + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids,operator," + + "operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { PreparedStatement insert = conn.prepareStatement(insertSql); PreparedStatement insertLog = conn.prepareStatement(insertLogSql); @@ -193,7 +200,8 @@ public class JsonSplitDao { insert.setInt(13, taskDefinitionLog.getFailRetryTimes()); insert.setInt(14, taskDefinitionLog.getFailRetryInterval()); insert.setInt(15, taskDefinitionLog.getTimeoutFlag().getCode()); - insert.setInt(16, taskDefinitionLog.getTimeoutNotifyStrategy() == null ? 0 : taskDefinitionLog.getTimeoutNotifyStrategy().getCode()); + insert.setInt(16, taskDefinitionLog.getTimeoutNotifyStrategy() == null ? 0 + : taskDefinitionLog.getTimeoutNotifyStrategy().getCode()); insert.setInt(17, taskDefinitionLog.getTimeout()); insert.setInt(18, taskDefinitionLog.getDelayTime()); insert.setString(19, taskDefinitionLog.getResourceIds()); @@ -216,7 +224,8 @@ public class JsonSplitDao { insertLog.setInt(13, taskDefinitionLog.getFailRetryTimes()); insertLog.setInt(14, taskDefinitionLog.getFailRetryInterval()); insertLog.setInt(15, taskDefinitionLog.getTimeoutFlag().getCode()); - insertLog.setInt(16, taskDefinitionLog.getTimeoutNotifyStrategy() == null ? 0 : taskDefinitionLog.getTimeoutNotifyStrategy().getCode()); + insertLog.setInt(16, taskDefinitionLog.getTimeoutNotifyStrategy() == null ? 0 + : taskDefinitionLog.getTimeoutNotifyStrategy().getCode()); insertLog.setInt(17, taskDefinitionLog.getTimeout()); insertLog.setInt(18, taskDefinitionLog.getDelayTime()); insertLog.setString(19, taskDefinitionLog.getResourceIds()); @@ -245,4 +254,4 @@ public class JsonSplitDao { ConnectionUtils.releaseResource(conn); } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java index f4d198e0a1..5ba498e861 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java @@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory; public class ProcessDefinitionDao { - public static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionDao.class); /** @@ -99,7 +98,8 @@ public class ProcessDefinitionDao { public List queryProcessDefinition(Connection conn) { List processDefinitions = new ArrayList<>(); - String sql = "SELECT id,code,project_code,user_id,locations,name,description,release_state,flag,create_time FROM t_ds_process_definition"; + String sql = + "SELECT id,code,project_code,user_id,locations,name,description,release_state,flag,create_time FROM t_ds_process_definition"; ResultSet rs = null; PreparedStatement pstmt = null; try { @@ -140,7 +140,8 @@ public class ProcessDefinitionDao { * @param processDefinitions processDefinitions * @param projectIdCodeMap projectIdCodeMap */ - public void updateProcessDefinitionCode(Connection conn, List processDefinitions, Map projectIdCodeMap) { + public void updateProcessDefinitionCode(Connection conn, List processDefinitions, + Map projectIdCodeMap) { String sql = "UPDATE t_ds_process_definition SET code=?, project_code=?, version=? where id=?"; try { for (ProcessDefinition processDefinition : processDefinitions) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java index 80a49fd42b..9d486d0077 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java @@ -68,7 +68,8 @@ public class ScheduleDao { * @param scheduleMap scheduleMap * @param processIdCodeMap processIdCodeMap */ - public void updateScheduleCode(Connection conn, Map scheduleMap, Map processIdCodeMap) { + public void updateScheduleCode(Connection conn, Map scheduleMap, + Map processIdCodeMap) { String sql = "UPDATE t_ds_schedules SET process_definition_code=?,timezone_id=?,environment_code=-1 where id=?"; try { Clock clock = Clock.systemDefaultZone(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java index f0fdeeea56..e6a7b00104 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java @@ -38,6 +38,7 @@ import com.google.common.base.Strings; * Metadata related common classes */ public class SchemaUtils { + private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); private SchemaUtils() { @@ -95,7 +96,8 @@ public class SchemaUtils { } } - // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number + // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger + // arrLength has a larger version number return schemaVersionArr.length > versionArr.length; } @@ -112,7 +114,8 @@ public class SchemaUtils { softVersion = Strings.nullToEmpty(softVersion).replaceAll("\\s+|\r|\n", ""); } catch (FileNotFoundException e) { logger.error(e.getMessage(), e); - throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); + throw new RuntimeException( + "Failed to get the product version description file. The file could not be found", e); } return softVersion; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java index 936b1d477e..e8a9fc7d82 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java @@ -18,16 +18,16 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class WorkerGroupDao { public static final Logger logger = LoggerFactory.getLogger(WorkerGroupDao.class); @@ -37,8 +37,8 @@ public class WorkerGroupDao { * @param conn jdbc connection * @return old worker group Map */ - public Map queryAllOldWorkerGroup(Connection conn){ - Map workerGroupMap = new HashMap<>(); + public Map queryAllOldWorkerGroup(Connection conn) { + Map workerGroupMap = new HashMap<>(); String sql = String.format("select id,name from t_ds_worker_group"); ResultSet rs = null; @@ -47,14 +47,14 @@ public class WorkerGroupDao { pstmt = conn.prepareStatement(sql); rs = pstmt.executeQuery(); - while (rs.next()){ + while (rs.next()) { int id = rs.getInt(1); String name = rs.getString(2); - workerGroupMap.put(id,name); + workerGroupMap.put(id, name); } } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql: " + sql, e); } finally { ConnectionUtils.releaseResource(rs, pstmt, conn); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java index 4bf4dbdb83..78a333488c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Component; */ @Component public class BeanContext implements ApplicationContextAware { + private static ApplicationContext applicationContext; public static ApplicationContext getApplicationContext() { @@ -42,7 +43,7 @@ public class BeanContext implements ApplicationContextAware { */ @SuppressWarnings("unchecked") public static T getBean(String name) throws BeansException { - return (T)applicationContext.getBean(name); + return (T) applicationContext.getBean(name); } /** @@ -56,7 +57,6 @@ public class BeanContext implements ApplicationContextAware { return applicationContext.getBean(clazz); } - /** * set applicationcontext * @param applicationContext applicationContext diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java index 928575b984..8440636ae8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java @@ -59,7 +59,9 @@ public class PostgreSQLPerformance extends BaseDBPerformance { } } - try (ResultSet rs3 = pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';")) { + try ( + ResultSet rs3 = + pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';")) { if (rs3.next()) { monitorRecord.setThreadsRunningConnections(rs3.getInt("count")); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java index 483c052b83..69d9788b32 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; * resource process definition utils */ public class ResourceProcessDefinitionUtils { + /** * get resource process map key is resource id,value is the set of process definition code * @@ -47,7 +48,8 @@ public class ResourceProcessDefinitionUtils { String[] resourceIds = ((String) resourceMap.get("resource_ids")) .split(","); - Set resourceIdSet = Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet()); + Set resourceIdSet = + Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet()); for (Integer resourceId : resourceIdSet) { Set codeSet; if (resourceResult.containsKey(resourceId)) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java index 1f6ffdfd09..431a022d59 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java @@ -54,7 +54,8 @@ public class TaskInstanceUtils { target.setAppLink(source.getAppLink()); target.setFlag(source.getFlag()); target.setDependency(source.getDependency()); - // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on task params, this is really a very bad practice. + // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on + // task params, this is really a very bad practice. target.setTaskParams(source.getTaskParams()); target.setSwitchDependency(source.getSwitchDependency()); target.setDuration(source.getDuration()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java index e2c2ccb31c..b453392e0e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.dao.utils; -import java.util.Date; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import java.util.Date; + /** * workflow utils */ @@ -33,11 +34,10 @@ public class WorkflowUtils { * @param processInstance workflow instance * @return workflow duration */ - public static String getWorkflowInstanceDuration (ProcessInstance processInstance) { - return processInstance.getState() != null && processInstance.getState().isFinished() ? - DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()) : - DateUtils.format2Duration(processInstance.getStartTime(), new Date()); + public static String getWorkflowInstanceDuration(ProcessInstance processInstance) { + return processInstance.getState() != null && processInstance.getState().isFinished() + ? DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()) + : DateUtils.format2Duration(processInstance.getStartTime(), new Date()); } } - diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java index 7c7e3f04ab..f2cb503d9b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java @@ -43,6 +43,7 @@ import org.springframework.transaction.annotation.Transactional; @Rollback @EnableTransactionManagement public class AlertDaoTest { + @Autowired private AlertDao alertDao; @@ -51,7 +52,7 @@ public class AlertDaoTest { Alert alert = new Alert(); alert.setTitle("Mysql Exception"); alert.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " - + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]"); + + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]"); alert.setAlertGroupId(1); alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); alertDao.addAlert(alert); @@ -63,7 +64,7 @@ public class AlertDaoTest { @Test public void testAddAlertSendStatus() { - int insertCount = alertDao.addAlertSendStatus(AlertStatus.EXECUTION_SUCCESS,"success",1,1); + int insertCount = alertDao.addAlertSendStatus(AlertStatus.EXECUTION_SUCCESS, "success", 1, 1); Assertions.assertEquals(1, insertCount); } @@ -75,9 +76,9 @@ public class AlertDaoTest { alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); long count = alertDao.listPendingAlerts() - .stream() - .filter(alert -> alert.getContent().contains(host)) - .count(); + .stream() + .filter(alert -> alert.getContent().contains(host)) + .count(); Assertions.assertEquals(1L, count); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java index 3844e9295f..31a40de612 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java @@ -29,11 +29,13 @@ public class ProcessDefinitionTest { public void getGlobalParamMapTest() { ProcessDefinition taskInstance = new ProcessDefinition(); - //sub process - taskInstance.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + // sub process + taskInstance.setGlobalParams( + "[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); taskInstance.getGlobalParamMap(); - Assertions.assertEquals("{selenium_global_parameters_1=selenium_global_parameters_value_1}",taskInstance.getGlobalParamMap().toString()); + Assertions.assertEquals("{selenium_global_parameters_1=selenium_global_parameters_value_1}", + taskInstance.getGlobalParamMap().toString()); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java index 392060afbd..cb68cf7494 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java @@ -42,19 +42,19 @@ public class TaskInstanceTest { public void testTaskInstanceIsSubProcess() { TaskInstance taskInstance = new TaskInstance(); - //sub process + // sub process taskInstance.setTaskType(TASK_TYPE_SUB_PROCESS); Assertions.assertTrue(taskInstance.isSubProcess()); - //not sub process + // not sub process taskInstance.setTaskType("HTTP"); Assertions.assertFalse(taskInstance.isSubProcess()); - //sub process + // sub process taskInstance.setTaskType(TASK_TYPE_CONDITIONS); Assertions.assertTrue(taskInstance.isConditionsTask()); - //sub process + // sub process taskInstance.setTaskType(TASK_TYPE_DEPENDENT); Assertions.assertTrue(taskInstance.isDependTask()); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java index bddc267545..b61975988a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java @@ -153,7 +153,6 @@ public class AccessTokenMapperTest extends BaseDaoTest { Assertions.assertEquals(Integer.valueOf(generalAccessTokenPage.getRecords().size()), emptySize); } - /** * test update */ @@ -161,7 +160,7 @@ public class AccessTokenMapperTest extends BaseDaoTest { public void testUpdate() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); - //update + // update accessToken.setToken("56789"); accessToken.setExpireTime(DateUtils.getCurrentDate()); accessToken.setUpdateTime(DateUtils.getCurrentDate()); @@ -200,7 +199,7 @@ public class AccessTokenMapperTest extends BaseDaoTest { * @throws Exception */ private Map createAccessTokens( - Integer count, String userName) throws Exception { + Integer count, String userName) throws Exception { User user = createUser(userName); @@ -249,7 +248,7 @@ public class AccessTokenMapperTest extends BaseDaoTest { * @return accessToken */ private AccessToken createAccessToken(Integer userId, String userName) { - //insertOne + // insertOne AccessToken accessToken = new AccessToken(); accessToken.setUserName(userName); accessToken.setUserId(userId); @@ -277,4 +276,4 @@ public class AccessTokenMapperTest extends BaseDaoTest { return createAccessToken(userId, null); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java index 21554e4117..54b112d8c2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; - import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.AlertGroup; @@ -54,19 +51,17 @@ public class AlertGroupMapperTest extends BaseDaoTest { } - /** * test selectById */ @Test public void testSelectById() { AlertGroup alertGroup = createAlertGroup(); - //query + // query AlertGroup targetAlert = alertGroupMapper.selectById(alertGroup.getId()); Assertions.assertEquals(alertGroup, targetAlert); } - /** * test page */ @@ -117,7 +112,6 @@ public class AlertGroupMapperTest extends BaseDaoTest { Assertions.assertEquals(alertGroup, resultAlertGroup); } - /** * test delete */ @@ -133,7 +127,6 @@ public class AlertGroupMapperTest extends BaseDaoTest { Assertions.assertNull(resultAlertGroup); } - /** * test query by groupname */ @@ -234,7 +227,7 @@ public class AlertGroupMapperTest extends BaseDaoTest { * @return AlertGroup map */ private Map createAlertGroups( - Integer count, String groupName) { + Integer count, String groupName) { Map alertGroupMap = new HashMap<>(); AlertGroup alertGroup = null; @@ -264,4 +257,4 @@ public class AlertGroupMapperTest extends BaseDaoTest { return alertGroupMap; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java index e807d69274..01f147e047 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java @@ -30,6 +30,7 @@ import org.springframework.beans.factory.annotation.Autowired; * AlertSendStatus mapper test */ public class AlertSendStatusMapperTest extends BaseDaoTest { + @Autowired private AlertSendStatusMapper alertSendStatusMapper; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java index e1a51c9008..a18b407344 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java @@ -67,4 +67,4 @@ public class AuditLogMapperTest extends BaseDaoTest { String resourceName = logMapper.queryResourceNameByType("USER", 1); Assertions.assertEquals("admin", resourceName); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 87e6ebe828..9548dd8223 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java index e8091769d9..63fd447470 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java @@ -38,7 +38,7 @@ public class DataSourceUserMapperTest extends BaseDaoTest { * @return DatasourceUser */ private DatasourceUser insertOne() { - //insertOne + // insertOne DatasourceUser dataSourceUser = new DatasourceUser(); dataSourceUser.setUserId(4); dataSourceUser.setDatasourceId(1010); @@ -54,9 +54,9 @@ public class DataSourceUserMapperTest extends BaseDaoTest { */ @Test public void testUpdate() { - //insertOne + // insertOne DatasourceUser dataSourceUser = insertOne(); - //update + // update dataSourceUser.setUpdateTime(new Date()); int update = dataSourceUserMapper.updateById(dataSourceUser); Assertions.assertEquals(update, 1); @@ -79,7 +79,7 @@ public class DataSourceUserMapperTest extends BaseDaoTest { @Test public void testQuery() { DatasourceUser dataSourceUser = insertOne(); - //query + // query List dataSources = dataSourceUserMapper.selectList(null); Assertions.assertNotEquals(dataSources.size(), 0); } @@ -103,4 +103,4 @@ public class DataSourceUserMapperTest extends BaseDaoTest { int delete = dataSourceUserMapper.deleteByDatasourceId(dataSourceUser.getDatasourceId()); Assertions.assertNotEquals(delete, 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java index ec4de0cf9f..b50095eed0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java @@ -16,20 +16,23 @@ */ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.DqRule; import org.apache.dolphinscheduler.dao.entity.DqRuleInputEntry; + +import java.util.List; + import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import java.util.List; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * DQC rule mapper UT. */ public class DqRuleInputEntryMapperTest extends BaseDaoTest { + @Autowired private DqRuleInputEntryMapper dqRuleInputEntryMapper; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java index d683c47ad8..2d41cf02e5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java @@ -56,7 +56,7 @@ public class EnvironmentWorkerGroupRelationMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private EnvironmentWorkerGroupRelation insertOne() { - //insertOne + // insertOne EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); relation.setEnvironmentCode(1L); relation.setWorkerGroup("default"); @@ -73,7 +73,7 @@ public class EnvironmentWorkerGroupRelationMapperTest extends BaseDaoTest { @Test public void testQuery() { insertOne(); - //query + // query List relations = environmentWorkerGroupRelationMapper.selectList(null); Assertions.assertEquals(relations.size(), 1); } @@ -81,14 +81,16 @@ public class EnvironmentWorkerGroupRelationMapperTest extends BaseDaoTest { @Test public void testQueryByEnvironmentCode() { EnvironmentWorkerGroupRelation relation = insertOne(); - List environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(1L); + List environmentWorkerGroupRelations = + environmentWorkerGroupRelationMapper.queryByEnvironmentCode(1L); Assertions.assertNotEquals(environmentWorkerGroupRelations.size(), 0); } @Test public void testQueryByWorkerGroupName() { EnvironmentWorkerGroupRelation relation = insertOne(); - List environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByWorkerGroupName("default"); + List environmentWorkerGroupRelations = + environmentWorkerGroupRelationMapper.queryByWorkerGroupName("default"); Assertions.assertNotEquals(environmentWorkerGroupRelations.size(), 0); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 6c717ca80c..d4c4b74991 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -43,7 +43,7 @@ public class ErrorCommandMapperTest extends BaseDaoTest { * @return ErrorCommand */ private ErrorCommand insertOne() { - //insertOne + // insertOne ErrorCommand errorCommand = new ErrorCommand(); errorCommand.setId(10101); errorCommand.setCommandType(CommandType.START_PROCESS); @@ -53,10 +53,6 @@ public class ErrorCommandMapperTest extends BaseDaoTest { return errorCommand; } - - - - /** * test query */ @@ -79,8 +75,7 @@ public class ErrorCommandMapperTest extends BaseDaoTest { List commandCounts = errorCommandMapper.countCommandState( null, null, - new Long[0] - ); + new Long[0]); Long[] projectCodeArray = new Long[2]; projectCodeArray[0] = processDefinition.getProjectCode(); @@ -88,8 +83,7 @@ public class ErrorCommandMapperTest extends BaseDaoTest { List commandCounts2 = errorCommandMapper.countCommandState( null, null, - projectCodeArray - ); + projectCodeArray); Assertions.assertNotEquals(commandCounts.size(), 0); Assertions.assertNotEquals(commandCounts2.size(), 0); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java index f96853e903..53e8744488 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java @@ -49,7 +49,8 @@ public class PluginDefineTest extends BaseDaoTest { @Test public void testQueryByNameAndType() { PluginDefine pluginDefine = createPluginDefine(); - PluginDefine pluginDefineSaved = pluginDefineMapper.queryByNameAndType(pluginDefine.getPluginName(), pluginDefine.getPluginType()); + PluginDefine pluginDefineSaved = + pluginDefineMapper.queryByNameAndType(pluginDefine.getPluginName(), pluginDefine.getPluginType()); Assertions.assertNotNull(pluginDefineSaved); Assertions.assertEquals(pluginDefineSaved.getPluginType(), pluginDefine.getPluginType()); Assertions.assertEquals(pluginDefineSaved.getPluginName(), pluginDefine.getPluginName()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java index a8013a41ff..904072a758 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java @@ -37,7 +37,7 @@ public class ProcessInstanceMapMapperTest extends BaseDaoTest { * @return ProcessInstanceMap */ private ProcessInstanceMap insertOne() { - //insertOne + // insertOne ProcessInstanceMap processInstanceMap = new ProcessInstanceMap(); processInstanceMap.setProcessInstanceId(0); processInstanceMap.setParentTaskInstanceId(0); @@ -51,9 +51,9 @@ public class ProcessInstanceMapMapperTest extends BaseDaoTest { */ @Test public void testUpdate() { - //insertOne + // insertOne ProcessInstanceMap processInstanceMap = insertOne(); - //update + // update processInstanceMap.setParentProcessInstanceId(1); int update = processInstanceMapMapper.updateById(processInstanceMap); Assertions.assertEquals(1, update); @@ -75,7 +75,7 @@ public class ProcessInstanceMapMapperTest extends BaseDaoTest { @Test public void testQuery() { ProcessInstanceMap processInstanceMap = insertOne(); - //query + // query List dataSources = processInstanceMapMapper.selectList(null); Assertions.assertNotEquals(dataSources.size(), 0); } @@ -102,8 +102,7 @@ public class ProcessInstanceMapMapperTest extends BaseDaoTest { processInstanceMap.setParentProcessInstanceId(100); processInstanceMapMapper.updateById(processInstanceMap); int delete = processInstanceMapMapper.deleteByParentProcessId( - processInstanceMap.getParentProcessInstanceId() - ); + processInstanceMap.getParentProcessInstanceId()); Assertions.assertEquals(1, delete); } @@ -119,10 +118,10 @@ public class ProcessInstanceMapMapperTest extends BaseDaoTest { processInstanceMapMapper.updateById(processInstanceMap); - List subIds = processInstanceMapMapper.querySubIdListByParentId(processInstanceMap.getParentProcessInstanceId()); + List subIds = + processInstanceMapMapper.querySubIdListByParentId(processInstanceMap.getParentProcessInstanceId()); Assertions.assertNotEquals(subIds.size(), 0); - } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java index 30839c5ea2..01a9b8b92c 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java @@ -25,12 +25,11 @@ import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import java.util.Date; import java.util.List; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -306,7 +305,8 @@ public class ProcessInstanceMapperTest extends BaseDaoTest { processInstanceMapper.updateById(processInstance); ProcessInstance processInstance1 = - processInstanceMapper.queryLastSchedulerProcess(processInstance.getProcessDefinitionCode(), null, null, processInstance.getTestFlag()); + processInstanceMapper.queryLastSchedulerProcess(processInstance.getProcessDefinitionCode(), null, null, + processInstance.getTestFlag()); Assertions.assertNotEquals(processInstance1, null); processInstanceMapper.deleteById(processInstance.getId()); } @@ -325,7 +325,8 @@ public class ProcessInstanceMapperTest extends BaseDaoTest { WorkflowExecutionStatus.SUBMITTED_SUCCESS.ordinal()}; ProcessInstance processInstance1 = processInstanceMapper - .queryLastRunningProcess(processInstance.getProcessDefinitionCode(), null, null, processInstance.getTestFlag(), stateArray); + .queryLastRunningProcess(processInstance.getProcessDefinitionCode(), null, null, + processInstance.getTestFlag(), stateArray); Assertions.assertNotEquals(processInstance1, null); processInstanceMapper.deleteById(processInstance.getId()); @@ -342,12 +343,14 @@ public class ProcessInstanceMapperTest extends BaseDaoTest { Date start = new Date(2019 - 1900, 1 - 1, 01, 0, 0, 0); Date end = new Date(2019 - 1900, 1 - 1, 01, 5, 0, 0); ProcessInstance processInstance1 = - processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, processInstance.getTestFlag()); + processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, + processInstance.getTestFlag()); Assertions.assertEquals(processInstance1.getId(), processInstance.getId()); start = new Date(2019 - 1900, 1 - 1, 01, 1, 0, 0); processInstance1 = - processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, processInstance.getTestFlag()); + processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, + processInstance.getTestFlag()); Assertions.assertNull(processInstance1); processInstanceMapper.deleteById(processInstance.getId()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java index 1316fd2645..3b82fd8e7f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java @@ -38,7 +38,7 @@ public class ProcessTaskRelationLogMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private ProcessTaskRelationLog insertOne() { - //insertOne + // insertOne ProcessTaskRelationLog processTaskRelationLog = new ProcessTaskRelationLog(); processTaskRelationLog.setName("def 1"); processTaskRelationLog.setProcessDefinitionVersion(1); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java index bb63978594..e83be99be0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java @@ -39,7 +39,7 @@ public class ProcessTaskRelationMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private ProcessTaskRelation insertOne() { - //insertOne + // insertOne ProcessTaskRelation processTaskRelation = new ProcessTaskRelation(); processTaskRelation.setName("def 1"); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java index 2dc27d273f..066995818a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java @@ -50,7 +50,7 @@ public class ProjectMapperTest extends BaseDaoTest { * @return Project */ private Project insertOne() { - //insertOne + // insertOne Project project = new Project(); project.setName("ut project"); project.setUserId(111); @@ -66,10 +66,10 @@ public class ProjectMapperTest extends BaseDaoTest { */ @Test public void testUpdate() { - //insertOne + // insertOne Project project = insertOne(); project.setCreateTime(new Date()); - //update + // update int update = projectMapper.updateById(project); Assertions.assertEquals(update, 1); } @@ -90,7 +90,7 @@ public class ProjectMapperTest extends BaseDaoTest { @Test public void testQuery() { Project project = insertOne(); - //query + // query List projects = projectMapper.selectList(null); Assertions.assertNotEquals(projects.size(), 0); } @@ -146,15 +146,13 @@ public class ProjectMapperTest extends BaseDaoTest { Page page = new Page(1, 3); IPage projectIPage = projectMapper.queryProjectListPaging( - page, - null, - null - ); + page, + null, + null); IPage projectIPage1 = projectMapper.queryProjectListPaging( - page, - null, - project.getName() - ); + page, + null, + project.getName()); Assertions.assertEquals(projectIPage.getTotal(), 1); Assertions.assertEquals(projectIPage1.getTotal(), 1); } @@ -192,8 +190,7 @@ public class ProjectMapperTest extends BaseDaoTest { Project project = insertOne(); List projects = projectMapper.queryProjectExceptUserId( - 100000 - ); + 100000); Assertions.assertNotEquals(projects.size(), 0); } @@ -230,4 +227,4 @@ public class ProjectMapperTest extends BaseDaoTest { Assertions.assertEquals(projects.size(), 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java index ee7d0c5e26..0307ba8697 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java @@ -38,7 +38,7 @@ public class ProjectUserMapperTest extends BaseDaoTest { * @return ProjectUser */ private ProjectUser insertOne() { - //insertOne + // insertOne ProjectUser projectUser = new ProjectUser(); projectUser.setProjectId(1010); projectUser.setUserId(111); @@ -51,10 +51,10 @@ public class ProjectUserMapperTest extends BaseDaoTest { */ @Test public void testUpdate() { - //insertOne + // insertOne ProjectUser projectUser = insertOne(); projectUser.setCreateTime(new Date()); - //update + // update int update = projectUserMapper.updateById(projectUser); Assertions.assertEquals(update, 1); } @@ -75,7 +75,7 @@ public class ProjectUserMapperTest extends BaseDaoTest { @Test public void testQuery() { ProjectUser projectUser = insertOne(); - //query + // query List projectUsers = projectUserMapper.selectList(null); Assertions.assertNotEquals(projectUsers.size(), 0); } @@ -98,8 +98,9 @@ public class ProjectUserMapperTest extends BaseDaoTest { @Test public void testQueryProjectRelation() { ProjectUser projectUser = insertOne(); - ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId()); + ProjectUser projectUser1 = + projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId()); Assertions.assertNotEquals(projectUser1, null); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 8dc3c6d7ab..94ed6907b1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -42,7 +42,7 @@ public class QueueMapperTest extends BaseDaoTest { * @return Queue */ private Queue insertOne() { - //insertOne + // insertOne Queue queue = new Queue(); queue.setQueueName("queue"); queue.setQueue("queue"); @@ -57,10 +57,10 @@ public class QueueMapperTest extends BaseDaoTest { */ @Test public void testUpdate() { - //insertOne + // insertOne Queue queue = insertOne(); queue.setCreateTime(new Date()); - //update + // update int update = queueMapper.updateById(queue); Assertions.assertEquals(1, update); } @@ -81,7 +81,7 @@ public class QueueMapperTest extends BaseDaoTest { @Test public void testQuery() { Queue queue = insertOne(); - //query + // query List queues = queueMapper.selectList(null); Assertions.assertNotEquals(queues.size(), 0); } @@ -93,12 +93,12 @@ public class QueueMapperTest extends BaseDaoTest { public void testQueryQueuePaging() { Queue queue = insertOne(); - Page page = new Page(1,3); + Page page = new Page(1, 3); - IPage queueIPage= queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), null); + IPage queueIPage = queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), null); Assertions.assertNotEquals(queueIPage.getTotal(), 0); - queueIPage= queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), queue.getQueueName()); + queueIPage = queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), queue.getQueueName()); Assertions.assertNotEquals(queueIPage.getTotal(), 0); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java index 9f6d1ead6d..dd57bc2aca 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java @@ -19,13 +19,14 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.ResourcesUser; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import java.util.Date; import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + public class ResourceUserMapperTest extends BaseDaoTest { @Autowired @@ -35,8 +36,8 @@ public class ResourceUserMapperTest extends BaseDaoTest { * insert * @return ResourcesUser */ - private ResourcesUser insertOne(){ - //insertOne + private ResourcesUser insertOne() { + // insertOne ResourcesUser resourcesUser = new ResourcesUser(); resourcesUser.setCreateTime(new Date()); resourcesUser.setUpdateTime(new Date()); @@ -51,11 +52,11 @@ public class ResourceUserMapperTest extends BaseDaoTest { * test update */ @Test - public void testUpdate(){ - //insertOne + public void testUpdate() { + // insertOne ResourcesUser queue = insertOne(); queue.setCreateTime(new Date()); - //update + // update int update = resourceUserMapper.updateById(queue); Assertions.assertEquals(1, update); } @@ -64,7 +65,7 @@ public class ResourceUserMapperTest extends BaseDaoTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { ResourcesUser queue = insertOne(); int delete = resourceUserMapper.deleteById(queue.getId()); Assertions.assertEquals(1, delete); @@ -76,7 +77,7 @@ public class ResourceUserMapperTest extends BaseDaoTest { @Test public void testQuery() { ResourcesUser queue = insertOne(); - //query + // query List queues = resourceUserMapper.selectList(null); Assertions.assertNotEquals(queues.size(), 0); } @@ -107,4 +108,4 @@ public class ResourceUserMapperTest extends BaseDaoTest { resourceIdArray); Assertions.assertNotEquals(delete, 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java index 93d65d20d0..e2d3747569 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java @@ -38,7 +38,7 @@ public class SessionMapperTest extends BaseDaoTest { * @return Session */ private Session insertOne() { - //insertOne + // insertOne Session session = new Session(); session.setId(UUID.randomUUID().toString()); session.setLastLoginTime(new Date()); @@ -51,11 +51,11 @@ public class SessionMapperTest extends BaseDaoTest { * test update */ @Test - public void testUpdate(){ - //insertOne + public void testUpdate() { + // insertOne Session session = insertOne(); session.setLastLoginTime(new Date()); - //update + // update int update = sessionMapper.updateById(session); Assertions.assertEquals(update, 1); } @@ -64,7 +64,7 @@ public class SessionMapperTest extends BaseDaoTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { Session session = insertOne(); int delete = sessionMapper.deleteById(session.getId()); Assertions.assertEquals(delete, 1); @@ -76,7 +76,7 @@ public class SessionMapperTest extends BaseDaoTest { @Test public void testQuery() { Session session = insertOne(); - //query + // query List sessions = sessionMapper.selectList(null); Assertions.assertNotEquals(sessions.size(), 0); } @@ -91,4 +91,4 @@ public class SessionMapperTest extends BaseDaoTest { Assertions.assertNotEquals(sessions.size(), 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java index 72dbd148d6..aa8fa7161e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java @@ -181,6 +181,6 @@ public class TaskDefinitionMapperTest extends BaseDaoTest { "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":null,\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendEmail\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; TaskDefinition definition = JSONUtils.parseObject(definitionJson, TaskDefinition.class); - Assertions.assertNull(definition.getTaskParamMap(),"Serialize the task definition success"); + Assertions.assertNull(definition.getTaskParamMap(), "Serialize the task definition success"); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java index 03d7bfc055..f45beb2a72 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java @@ -26,7 +26,6 @@ import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java index 849ffa9c8d..be71c3561f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java @@ -183,8 +183,8 @@ public class TaskInstanceMapperTest extends BaseDaoTest { task2.setFlag(Flag.NO); taskInstanceMapper.updateById(task2); List taskInstances1 = taskInstanceMapper.findValidTaskListByProcessId(task.getProcessInstanceId(), - Flag.NO, - processInstance.getTestFlag()); + Flag.NO, + processInstance.getTestFlag()); taskInstanceMapper.deleteById(task2.getId()); taskInstanceMapper.deleteById(task.getId()); Assertions.assertNotEquals(taskInstances.size(), 0); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java index 225a997de0..fc6c2479d0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java @@ -43,8 +43,8 @@ public class TenantMapperTest extends BaseDaoTest { * insert * @return Tenant */ - private Tenant insertOne(){ - //insertOne + private Tenant insertOne() { + // insertOne Tenant tenant = new Tenant(); tenant.setCreateTime(new Date()); tenant.setUpdateTime(new Date()); @@ -57,11 +57,11 @@ public class TenantMapperTest extends BaseDaoTest { * test update */ @Test - public void testUpdate(){ - //insertOne + public void testUpdate() { + // insertOne Tenant tenant = insertOne(); tenant.setUpdateTime(new Date()); - //update + // update int update = tenantMapper.updateById(tenant); Assertions.assertEquals(1, update); } @@ -70,7 +70,7 @@ public class TenantMapperTest extends BaseDaoTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { Tenant tenant = insertOne(); int delete = tenantMapper.deleteById(tenant.getId()); Assertions.assertEquals(1, delete); @@ -82,7 +82,7 @@ public class TenantMapperTest extends BaseDaoTest { @Test public void testQuery() { Tenant tenant = insertOne(); - //query + // query List tenants = tenantMapper.selectList(null); Assertions.assertNotEquals(tenants.size(), 0); } @@ -98,7 +98,6 @@ public class TenantMapperTest extends BaseDaoTest { queue.setQueue("ut queue"); queueMapper.insert(queue); - Tenant tenant = insertOne(); tenant.setQueueId(queue.getId()); tenantMapper.updateById(tenant); @@ -136,8 +135,9 @@ public class TenantMapperTest extends BaseDaoTest { tenantMapper.updateById(tenant); Page page = new Page(1, 3); - //tenant.getTenantCode() used instead of tenant.getTenantName() - IPage tenantIPage = tenantMapper.queryTenantPaging(page, Collections.singletonList(tenant.getId()), tenant.getTenantCode()); + // tenant.getTenantCode() used instead of tenant.getTenantName() + IPage tenantIPage = + tenantMapper.queryTenantPaging(page, Collections.singletonList(tenant.getId()), tenant.getTenantCode()); Assertions.assertNotEquals(tenantIPage.getTotal(), 0); } @@ -148,4 +148,4 @@ public class TenantMapperTest extends BaseDaoTest { insertOne(); Assertions.assertTrue(tenantMapper.existTenant(tenantCode)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java index 16aae713d2..ec90fa2c82 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java @@ -45,7 +45,7 @@ public class UDFUserMapperTest extends BaseDaoTest { * insert * @return UDFUser */ - private UDFUser insertOne(){ + private UDFUser insertOne() { UDFUser udfUser = new UDFUser(); udfUser.setUdfId(1); udfUser.setUserId(1); @@ -61,7 +61,7 @@ public class UDFUserMapperTest extends BaseDaoTest { * @param udfFunc udfFunc * @return UDFUser */ - private UDFUser insertOne(User user,UdfFunc udfFunc){ + private UDFUser insertOne(User user, UdfFunc udfFunc) { UDFUser udfUser = new UDFUser(); udfUser.setUdfId(udfFunc.getId()); udfUser.setUserId(user.getId()); @@ -71,12 +71,11 @@ public class UDFUserMapperTest extends BaseDaoTest { return udfUser; } - /** * insert one user * @return User */ - private User insertOneUser(){ + private User insertOneUser() { User user = new User(); user.setUserName("user1"); user.setUserPassword("1"); @@ -94,7 +93,7 @@ public class UDFUserMapperTest extends BaseDaoTest { * insert one udf * @return UdfFunc */ - private UdfFunc insertOneUdfFunc(){ + private UdfFunc insertOneUdfFunc() { UdfFunc udfFunc = new UdfFunc(); udfFunc.setFuncName("dolphin_udf_func"); udfFunc.setClassName("org.apache.dolphinscheduler.test.mr"); @@ -111,12 +110,12 @@ public class UDFUserMapperTest extends BaseDaoTest { * test update */ @Test - public void testUpdate(){ - //insertOneUser + public void testUpdate() { + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); udfUser.setUserId(2); udfUser.setUdfId(2); @@ -129,12 +128,12 @@ public class UDFUserMapperTest extends BaseDaoTest { * test delete */ @Test - public void testDelete(){ - //insertOneUser + public void testDelete() { + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); int delete = udfUserMapper.deleteById(udfUser.getId()); Assertions.assertEquals(delete, 1); @@ -144,10 +143,10 @@ public class UDFUserMapperTest extends BaseDaoTest { * test query */ @Test - public void testQuery(){ - //insertOne + public void testQuery() { + // insertOne UDFUser udfUser = insertOne(); - //query + // query List udfUserList = udfUserMapper.selectList(null); Assertions.assertNotEquals(udfUserList.size(), 0); } @@ -157,11 +156,11 @@ public class UDFUserMapperTest extends BaseDaoTest { */ @Test public void testDeleteByUserId() { - //insertOneUser + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); int delete = udfUserMapper.deleteByUserId(user.getId()); Assertions.assertEquals(delete, 1); @@ -173,13 +172,13 @@ public class UDFUserMapperTest extends BaseDaoTest { */ @Test public void testDeleteByUdfFuncId() { - //insertOneUser + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); int delete = udfUserMapper.deleteByUdfFuncId(udfFunc.getId()); Assertions.assertEquals(delete, 1); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java index 600fda93f9..b9a1a0b01f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java @@ -38,6 +38,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; public class UserMapperTest extends BaseDaoTest { + @Autowired private UserMapper userMapper; @@ -119,7 +120,7 @@ public class UserMapperTest extends BaseDaoTest { * @return AlertGroup */ private AlertGroup insertOneAlertGroup() { - //insertOne + // insertOne AlertGroup alertGroup = new AlertGroup(); alertGroup.setGroupName("alert group 1"); alertGroup.setDescription("alert test1"); @@ -137,7 +138,7 @@ public class UserMapperTest extends BaseDaoTest { * @return AccessToken */ private AccessToken insertOneAccessToken(User user) { - //insertOne + // insertOne AccessToken accessToken = new AccessToken(); accessToken.setUserId(user.getId()); accessToken.setToken("secrettoken"); @@ -201,9 +202,9 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testUpdate() { - //insertOne + // insertOne User user = insertOne(); - //update + // update user.setEmail("xx-update@126.com"); user.setUserName("user1_update"); user.setUserType(UserType.ADMIN_USER); @@ -216,9 +217,9 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testDelete() { - //insertOne + // insertOne User user = insertOne(); - //delete + // delete int delete = userMapper.deleteById(user.getId()); Assertions.assertEquals(delete, 1); } @@ -228,9 +229,9 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testQuery() { - //insertOne + // insertOne User user = insertOne(); - //query + // query List userList = userMapper.selectList(null); Assertions.assertNotEquals(userList.size(), 0); } @@ -240,26 +241,25 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testQueryAllGeneralUser() { - //insertOne + // insertOne User user = insertOne(); - //queryAllGeneralUser + // queryAllGeneralUser List userList = userMapper.queryAllGeneralUser(); Assertions.assertNotEquals(userList.size(), 0); } - /** * test page */ @Test public void testQueryUserPaging() { - //insertOneQueue + // insertOneQueue Queue queue = insertOneQueue(); - //insertOneTenant + // insertOneTenant Tenant tenant = insertOneTenant(); - //insertOne + // insertOne User user = insertOne(queue, tenant); - //queryUserPaging + // queryUserPaging Page page = new Page(1, 3); IPage userIPage = userMapper.queryUserPaging(page, user.getUserName()); Assertions.assertNotEquals(userIPage.getTotal(), 0); @@ -270,11 +270,11 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testQueryDetailsById() { - //insertOneQueue and insertOneTenant + // insertOneQueue and insertOneTenant Queue queue = insertOneQueue(); Tenant tenant = insertOneTenant(queue); User user = insertOne(queue, tenant); - //queryDetailsById + // queryDetailsById User queryUser = userMapper.queryDetailsById(user.getId()); Assertions.assertEquals(user.getUserName(), queryUser.getUserName()); } @@ -284,11 +284,11 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testQueryTenantCodeByUserId() { - //insertOneTenant + // insertOneTenant Tenant tenant = insertOneTenant(); - //insertOne + // insertOne User user = insertOne(tenant); - //queryTenantCodeByUserId + // queryTenantCodeByUserId User queryUser = userMapper.queryTenantCodeByUserId(user.getId()); Assertions.assertEquals(queryUser, user); } @@ -298,11 +298,11 @@ public class UserMapperTest extends BaseDaoTest { */ @Test public void testQueryUserByToken() { - //insertOne + // insertOne User user = insertOne(); - //insertOneAccessToken + // insertOneAccessToken AccessToken accessToken = insertOneAccessToken(user); - //queryUserByToken + // queryUserByToken User userToken = userMapper.queryUserByToken(accessToken.getToken(), new Date()); Assertions.assertEquals(userToken, user); @@ -310,7 +310,7 @@ public class UserMapperTest extends BaseDaoTest { @Test public void selectByIds() { - //insertOne + // insertOne User user = insertOne(); List userIds = new ArrayList<>(); userIds.add(user.getId()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java index a607092ee7..1d0083d11b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java @@ -54,7 +54,7 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private ProcessTaskRelation insertOneProcessTaskRelation() { - //insertOne + // insertOne ProcessTaskRelation processTaskRelation = new ProcessTaskRelation(); processTaskRelation.setName("def 1"); @@ -75,7 +75,7 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { * */ private void insertOneProcessDefinition() { - //insertOne + // insertOne ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setCode(1L); processDefinition.setName("def 1"); @@ -91,7 +91,7 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { * */ private void insertOneSchedule(int id) { - //insertOne + // insertOne Schedule schedule = new Schedule(); schedule.setStartTime(new Date()); schedule.setEndTime(new Date()); @@ -111,15 +111,17 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { ProcessDefinition processDefinition = processDefinitionMapper.queryByCode(1L); insertOneSchedule(processDefinition.getId()); - List workFlowLineages = workFlowLineageMapper.queryWorkFlowLineageByName(processDefinition.getProjectCode(), processDefinition.getName()); + List workFlowLineages = workFlowLineageMapper + .queryWorkFlowLineageByName(processDefinition.getProjectCode(), processDefinition.getName()); Assertions.assertNotEquals(workFlowLineages.size(), 0); } @Test public void testQueryCodeRelation() { ProcessTaskRelation processTaskRelation = insertOneProcessTaskRelation(); - List workFlowLineages = workFlowLineageMapper.queryProcessLineageByCode(processTaskRelation.getProjectCode(), - processTaskRelation.getProcessDefinitionCode()); + List workFlowLineages = + workFlowLineageMapper.queryProcessLineageByCode(processTaskRelation.getProjectCode(), + processTaskRelation.getProcessDefinitionCode()); Assertions.assertNotEquals(workFlowLineages.size(), 0); } @@ -128,7 +130,8 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { insertOneProcessDefinition(); ProcessDefinition processDefinition = processDefinitionMapper.queryByCode(1L); insertOneProcessTaskRelation(); - List workFlowLineages = workFlowLineageMapper.queryProcessLineage(processDefinition.getProjectCode()); + List workFlowLineages = + workFlowLineageMapper.queryProcessLineage(processDefinition.getProjectCode()); Assertions.assertNotEquals(workFlowLineages.size(), 0); } @@ -137,7 +140,8 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { insertOneProcessDefinition(); ProcessDefinition processDefinition = processDefinitionMapper.queryByCode(1L); insertOneSchedule(processDefinition.getId()); - WorkFlowLineage workFlowLineages = workFlowLineageMapper.queryWorkFlowLineageByCode(processDefinition.getProjectCode(), processDefinition.getCode()); + WorkFlowLineage workFlowLineages = workFlowLineageMapper + .queryWorkFlowLineageByCode(processDefinition.getProjectCode(), processDefinition.getCode()); Assertions.assertNotNull(workFlowLineages); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java index 6ddff0b560..afd474193a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.dao.BaseDaoTest; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; + import java.util.Date; import java.util.List; -import javax.annotation.Resource; -import org.apache.dolphinscheduler.dao.BaseDaoTest; -import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import javax.annotation.Resource; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -49,7 +50,7 @@ public class WorkerGroupMapperTest extends BaseDaoTest { */ public void clearTestData() { workerGroupMapper.queryAllWorkerGroup() - .forEach(workerGroup -> workerGroupMapper.deleteById(workerGroup.getId())); + .forEach(workerGroup -> workerGroupMapper.deleteById(workerGroup.getId())); } /** @@ -113,4 +114,4 @@ public class WorkerGroupMapperTest extends BaseDaoTest { int delete = workerGroupMapper.deleteById(workerGroup.getId()); Assertions.assertEquals(1, delete); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java index 9480aa4404..6a1599360b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java @@ -29,32 +29,32 @@ import org.springframework.test.context.ActiveProfiles; @ActiveProfiles("h2") public class ProcessDefinitionDaoTest { + @Autowired private DataSource dataSource; final ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); @Test public void testQueryAllProcessDefinition() { - //Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); - //assertThat(processDefinitionJsonMap.size(),greaterThanOrEqualTo(0)); + // Map processDefinitionJsonMap = + // processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + // assertThat(processDefinitionJsonMap.size(),greaterThanOrEqualTo(0)); } @Test public void testUpdateProcessDefinitionJson() { - Map processDefinitionJsonMap = new HashMap<>(); - processDefinitionJsonMap.put(1,"test"); - //processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),processDefinitionJsonMap); + Map processDefinitionJsonMap = new HashMap<>(); + processDefinitionJsonMap.put(1, "test"); + // processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),processDefinitionJsonMap); } @Test public void testQueryAllProcessDefinitionException() { - //processDefinitionDao.queryAllProcessDefinition(null); + // processDefinitionDao.queryAllProcessDefinition(null); } @Test public void testUpdateProcessDefinitionJsonException() { - Assertions.assertThrows(Exception.class, () -> - processDefinitionDao.updateProcessDefinitionJson(null, null) - ); + Assertions.assertThrows(Exception.class, () -> processDefinitionDao.updateProcessDefinitionJson(null, null)); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java index 3b8885ad40..cc3bfeaea9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java @@ -71,6 +71,6 @@ public class SchemaUtilsTest { } catch (IOException ex) { Assertions.fail(ex.getMessage()); } - Assertions.assertFalse(CollectionUtils.isEmpty(list),"Can not find any schema files"); + Assertions.assertFalse(CollectionUtils.isEmpty(list), "Can not find any schema files"); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java index 3f1e1eb709..0c36d5f136 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java @@ -18,8 +18,6 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.dao.DaoConfiguration; -import java.util.Map; - import javax.sql.DataSource; import org.junit.jupiter.api.Assertions; @@ -36,15 +34,16 @@ import org.springframework.test.context.ActiveProfiles; @ExtendWith(MockitoExtension.class) @SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) public class WorkerGroupDaoTest { + @Autowired protected DataSource dataSource; @Test public void testQueryQueryAllOldWorkerGroupException() throws Exception { Assertions.assertThrows(Exception.class, () -> { - WorkerGroupDao workerGroupDao = new WorkerGroupDao(); - workerGroupDao.queryAllOldWorkerGroup(null); - }); + WorkerGroupDao workerGroupDao = new WorkerGroupDao(); + workerGroupDao.queryAllOldWorkerGroup(null); + }); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java index 01135f3d43..d12c5e19b7 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java @@ -16,26 +16,27 @@ */ package org.apache.dolphinscheduler.dao.utils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + /** * resource process definition utils test */ public class ResourceProcessDefinitionUtilsTest { + @Test - public void getResourceProcessDefinitionMapTest(){ - List> mapList = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("code",1L); - map.put("resource_ids","1,2,3"); + public void getResourceProcessDefinitionMapTest() { + List> mapList = new ArrayList<>(); + Map map = new HashMap<>(); + map.put("code", 1L); + map.put("resource_ids", "1,2,3"); mapList.add(map); Assertions.assertNotNull(ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(mapList)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java index 497c2fb881..cb74aeb44e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java @@ -40,4 +40,4 @@ class TaskInstanceUtilsTest { Assertions.assertEquals(target.getId(), source.getId()); Assertions.assertEquals(target.getName(), source.getName()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java index 985066cbdb..13b839849d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java @@ -17,10 +17,12 @@ package org.apache.dolphinscheduler.dao.utils; -import java.util.Date; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; + +import java.util.Date; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -48,4 +50,4 @@ class WorkflowUtilsTest { String successDuration = WorkflowUtils.getWorkflowInstanceDuration(processInstance); Assertions.assertEquals("1d 1h 10m 10s", successDuration); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java index cf37df8f7b..191e557f44 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java @@ -50,7 +50,8 @@ public class DataQualityApplication { String dataQualityParameter = args[0]; - DataQualityConfiguration dataQualityConfiguration = JsonUtils.fromJson(dataQualityParameter,DataQualityConfiguration.class); + DataQualityConfiguration dataQualityConfiguration = + JsonUtils.fromJson(dataQualityParameter, DataQualityConfiguration.class); if (dataQualityConfiguration == null) { logger.info("DataQualityConfiguration is null"); System.exit(-1); @@ -60,13 +61,14 @@ public class DataQualityApplication { EnvConfig envConfig = dataQualityConfiguration.getEnvConfig(); Config config = new Config(envConfig.getConfig()); - config.put("type",envConfig.getType()); + config.put("type", envConfig.getType()); if (Strings.isNullOrEmpty(config.getString(SPARK_APP_NAME))) { - config.put(SPARK_APP_NAME,dataQualityConfiguration.getName()); + config.put(SPARK_APP_NAME, dataQualityConfiguration.getName()); } SparkRuntimeEnvironment sparkRuntimeEnvironment = new SparkRuntimeEnvironment(config); - DataQualityContext dataQualityContext = new DataQualityContext(sparkRuntimeEnvironment,dataQualityConfiguration); + DataQualityContext dataQualityContext = + new DataQualityContext(sparkRuntimeEnvironment, dataQualityConfiguration); dataQualityContext.execute(); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java index 647016abf6..60c5b810c3 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java @@ -32,12 +32,12 @@ public class BaseConfig implements IConfig { private String type; @JsonProperty("config") - private Map config; + private Map config; public BaseConfig() { } - public BaseConfig(String type, Map config) { + public BaseConfig(String type, Map config) { this.type = type; this.config = config; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java index fbf7e70f84..0bfe9e8df1 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java @@ -28,13 +28,13 @@ import java.util.Set; */ public class Config { - private Map configuration = new HashMap<>(); + private Map configuration = new HashMap<>(); public Config() { } - public Config(Map configuration) { + public Config(Map configuration) { if (configuration != null) { this.configuration = configuration; } @@ -45,7 +45,7 @@ public class Config { } public List getStringList(String key) { - return (List)configuration.get(key); + return (List) configuration.get(key); } public Integer getInt(String key) { @@ -81,7 +81,7 @@ public class Config { } public void put(String key, Object value) { - this.configuration.put(key,value); + this.configuration.put(key, value); } public void merge(Map configuration) { diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java index 6ffd955663..43a4a271c4 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java @@ -43,7 +43,8 @@ public class DataQualityConfiguration implements IConfig { @JsonProperty("writers") private List writerConfigs; - public DataQualityConfiguration() {} + public DataQualityConfiguration() { + } public DataQualityConfiguration(String name, EnvConfig envConfig, @@ -114,7 +115,7 @@ public class DataQualityConfiguration implements IConfig { } Preconditions.checkArgument(writerConfigs != null, "writer config should not be empty"); - for (WriterConfig writerConfig :writerConfigs) { + for (WriterConfig writerConfig : writerConfigs) { writerConfig.validate(); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java index 91d6854992..b37c857338 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java @@ -27,8 +27,8 @@ public class EnvConfig extends BaseConfig { public EnvConfig() { } - public EnvConfig(String type, Map config) { - super(type,config); + public EnvConfig(String type, Map config) { + super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java index 72aa556865..900be619ad 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java @@ -24,9 +24,10 @@ import java.util.Map; */ public class ReaderConfig extends BaseConfig { - public ReaderConfig() {} + public ReaderConfig() { + } - public ReaderConfig(String type, Map config) { + public ReaderConfig(String type, Map config) { super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java index 59d24c6b3b..47a540760b 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java @@ -24,9 +24,10 @@ import java.util.Map; */ public class TransformerConfig extends BaseConfig { - public TransformerConfig() {} + public TransformerConfig() { + } - public TransformerConfig(String type, Map config) { + public TransformerConfig(String type, Map config) { super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java index 75c5049311..9132c8930f 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java @@ -24,9 +24,10 @@ import java.util.Map; */ public class WriterConfig extends BaseConfig { - public WriterConfig() {} + public WriterConfig() { + } - public WriterConfig(String type, Map config) { + public WriterConfig(String type, Map config) { super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java index 494bbad364..b702461533 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java @@ -50,16 +50,16 @@ public class DataQualityContext { public void execute() throws DataQualityException { List readers = ReaderFactory .getInstance() - .getReaders(this.sparkRuntimeEnvironment,dataQualityConfiguration.getReaderConfigs()); + .getReaders(this.sparkRuntimeEnvironment, dataQualityConfiguration.getReaderConfigs()); List transformers = TransformerFactory .getInstance() - .getTransformer(this.sparkRuntimeEnvironment,dataQualityConfiguration.getTransformerConfigs()); + .getTransformer(this.sparkRuntimeEnvironment, dataQualityConfiguration.getTransformerConfigs()); List writers = WriterFactory .getInstance() - .getWriters(this.sparkRuntimeEnvironment,dataQualityConfiguration.getWriterConfigs()); + .getWriters(this.sparkRuntimeEnvironment, dataQualityConfiguration.getWriterConfigs()); if (sparkRuntimeEnvironment.isBatch()) { - sparkRuntimeEnvironment.getBatchExecution().execute(readers,transformers,writers); + sparkRuntimeEnvironment.getBatchExecution().execute(readers, transformers, writers); } else { throw new DataQualityException("stream mode is not supported now"); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java index b541367d96..e0cc02b3d8 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java @@ -21,6 +21,7 @@ package org.apache.dolphinscheduler.data.quality.enums; * ReaderType */ public enum ReaderType { + /** * JDBC * HIVE @@ -29,7 +30,7 @@ public enum ReaderType { HIVE; public static ReaderType getType(String name) { - for (ReaderType type: ReaderType.values()) { + for (ReaderType type : ReaderType.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java index b98adc4163..07744d9eec 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java @@ -21,13 +21,14 @@ package org.apache.dolphinscheduler.data.quality.enums; * TransformerType */ public enum TransformerType { + /** * JDBC */ SQL; public static TransformerType getType(String name) { - for (TransformerType type: TransformerType.values()) { + for (TransformerType type : TransformerType.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java index 369de959be..f46430a4f6 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java @@ -21,6 +21,7 @@ package org.apache.dolphinscheduler.data.quality.enums; * WriterType */ public enum WriterType { + /** * JDBC */ @@ -29,7 +30,7 @@ public enum WriterType { HDFS_FILE; public static WriterType getType(String name) { - for (WriterType type: WriterType.values()) { + for (WriterType type : WriterType.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java index 1644b2adde..527e10397a 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java @@ -25,6 +25,7 @@ import java.util.List; * Execution */ public interface Execution { + /** * execute * @param readers readers diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java index b805253dc3..cf55c1c23d 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java @@ -49,12 +49,12 @@ public class SparkBatchExecution implements Execution ds = readers.get(0).read(environment); - for (BatchTransformer tf:transformers) { + for (BatchTransformer tf : transformers) { ds = executeTransformer(environment, tf, ds); registerTransformTempView(tf, ds); } - for (BatchWriter sink: writers) { + for (BatchWriter sink : writers) { executeWriter(environment, sink, ds); } } @@ -77,18 +77,20 @@ public class SparkBatchExecution implements Execution executeTransformer(SparkRuntimeEnvironment environment, BatchTransformer transformer, Dataset dataset) { + private Dataset executeTransformer(SparkRuntimeEnvironment environment, BatchTransformer transformer, + Dataset dataset) { Config config = transformer.getConfig(); Dataset inputDataset; Dataset outputDataset = null; if (Boolean.TRUE.equals(config.has(INPUT_TABLE))) { String[] tableNames = config.getString(INPUT_TABLE).split(","); - for (String sourceTableName: tableNames) { + for (String sourceTableName : tableNames) { inputDataset = environment.sparkSession().read().table(sourceTableName); if (outputDataset == null) { diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java index b746392bda..2e46a3c44b 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java @@ -53,8 +53,8 @@ public class SparkRuntimeEnvironment { private SparkConf createSparkConf() { SparkConf conf = new SparkConf(); this.config.entrySet() - .forEach(entry -> conf.set(entry.getKey(), String.valueOf(entry.getValue()))); - conf.set("spark.sql.crossJoin.enabled","true"); + .forEach(entry -> conf.set(entry.getKey(), String.valueOf(entry.getValue()))); + conf.set("spark.sql.crossJoin.enabled", "true"); return conf; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java index ee5cae205a..c85c2cf50d 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java @@ -45,8 +45,8 @@ public interface Component { if (!nonExistsOptions.isEmpty()) { return new ValidateResult( false, - nonExistsOptions.stream().map(option -> - "[" + option + "]").collect(Collectors.joining(",")) + " is not exist"); + nonExistsOptions.stream().map(option -> "[" + option + "]").collect(Collectors.joining(",")) + + " is not exist"); } else { return new ValidateResult(true, ""); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java index cd76edc364..4022077a01 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java @@ -57,7 +57,7 @@ public class HiveReader implements BatchReader { @Override public void prepare(SparkRuntimeEnvironment prepareEnv) { if (Strings.isNullOrEmpty(config.getString(SQL))) { - config.put(SQL,"select * from " + config.getString(DATABASE) + "." + config.getString(TABLE)); + config.put(SQL, "select * from " + config.getString(DATABASE) + "." + config.getString(TABLE)); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java index 0484c15fa2..96f980fcb7 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java @@ -85,8 +85,8 @@ public class JdbcReader implements BatchReader { Config jdbcConfig = ConfigUtils.extractSubConfig(config, JDBC + DOTS, false); if (!config.isEmpty()) { - Map optionMap = new HashMap<>(16); - jdbcConfig.entrySet().forEach(x -> optionMap.put(x.getKey(),String.valueOf(x.getValue()))); + Map optionMap = new HashMap<>(16); + jdbcConfig.entrySet().forEach(x -> optionMap.put(x.getKey(), String.valueOf(x.getValue()))); reader.options(optionMap); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java index 9d9eb3d8e7..cafe3decdb 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java @@ -33,6 +33,7 @@ import java.util.List; public class ReaderFactory { private static class Singleton { + static ReaderFactory instance = new ReaderFactory(); } @@ -40,7 +41,8 @@ public class ReaderFactory { return Singleton.instance; } - public List getReaders(SparkRuntimeEnvironment sparkRuntimeEnvironment, List readerConfigs) throws DataQualityException { + public List getReaders(SparkRuntimeEnvironment sparkRuntimeEnvironment, + List readerConfigs) throws DataQualityException { List readerList = new ArrayList<>(); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java index cb5e45eaed..bc5a19eac9 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java @@ -33,6 +33,7 @@ import java.util.List; public class TransformerFactory { private static class Singleton { + static TransformerFactory instance = new TransformerFactory(); } @@ -40,11 +41,12 @@ public class TransformerFactory { return Singleton.instance; } - public List getTransformer(SparkRuntimeEnvironment sparkRuntimeEnvironment, List transformerConfigs) throws DataQualityException { + public List getTransformer(SparkRuntimeEnvironment sparkRuntimeEnvironment, + List transformerConfigs) throws DataQualityException { List transformers = new ArrayList<>(); - for (TransformerConfig transformerConfig:transformerConfigs) { + for (TransformerConfig transformerConfig : transformerConfigs) { BatchTransformer transformer = getTransformer(transformerConfig); if (transformer != null) { transformer.validateConfig(); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java index f1c098965c..0010c4c6e6 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java @@ -64,7 +64,7 @@ public class JdbcWriter implements BatchWriter { @Override public void prepare(SparkRuntimeEnvironment prepareEnv) { if (Strings.isNullOrEmpty(config.getString(SAVE_MODE))) { - config.put(SAVE_MODE,APPEND); + config.put(SAVE_MODE, APPEND); } } @@ -75,13 +75,13 @@ public class JdbcWriter implements BatchWriter { } data.write() - .format(JDBC) - .option(DRIVER,config.getString(DRIVER)) - .option(URL,config.getString(URL)) - .option(DB_TABLE, config.getString(TABLE)) - .option(USER, config.getString(USER)) - .option(PASSWORD, config.getString(PASSWORD)) - .mode(config.getString(SAVE_MODE)) - .save(); + .format(JDBC) + .option(DRIVER, config.getString(DRIVER)) + .option(URL, config.getString(URL)) + .option(DB_TABLE, config.getString(TABLE)) + .option(USER, config.getString(USER)) + .option(PASSWORD, config.getString(PASSWORD)) + .mode(config.getString(SAVE_MODE)) + .save(); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java index f907748a4d..e9fc2af6c8 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java @@ -35,6 +35,7 @@ import java.util.List; public class WriterFactory { private static class Singleton { + static WriterFactory instance = new WriterFactory(); } @@ -42,11 +43,12 @@ public class WriterFactory { return Singleton.instance; } - public List getWriters(SparkRuntimeEnvironment sparkRuntimeEnvironment, List writerConfigs) throws DataQualityException { + public List getWriters(SparkRuntimeEnvironment sparkRuntimeEnvironment, + List writerConfigs) throws DataQualityException { List writerList = new ArrayList<>(); - for (WriterConfig writerConfig:writerConfigs) { + for (WriterConfig writerConfig : writerConfigs) { BatchWriter writer = getWriter(writerConfig); if (writer != null) { writer.validateConfig(); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java index fbe311fe2a..56ad58044e 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java @@ -59,11 +59,11 @@ public abstract class BaseFileWriter implements BatchWriter { @Override public void prepare(SparkRuntimeEnvironment prepareEnv) { - Map defaultConfig = new HashMap<>(); + Map defaultConfig = new HashMap<>(); defaultConfig.put(PARTITION_BY, Collections.emptyList()); - defaultConfig.put(SAVE_MODE,"error"); - defaultConfig.put(SERIALIZER,"csv"); + defaultConfig.put(SAVE_MODE, "error"); + defaultConfig.put(SERIALIZER, "csv"); config.merge(defaultConfig); } @@ -75,7 +75,8 @@ public abstract class BaseFileWriter implements BatchWriter { if (dir.startsWith("/") || uriInAllowedSchema(dir, allowedUri)) { return new ValidateResult(true, ""); } else { - return new ValidateResult(false, "invalid path URI, please set the following allowed schemas: " + String.join(",", allowedUri)); + return new ValidateResult(false, + "invalid path URI, please set the following allowed schemas: " + String.join(",", allowedUri)); } } else { return new ValidateResult(false, "please specify [path] as non-empty string"); @@ -101,8 +102,8 @@ public abstract class BaseFileWriter implements BatchWriter { Config fileConfig = ConfigUtils.extractSubConfig(config, "options.", false); if (fileConfig.isNotEmpty()) { - Map optionMap = new HashMap<>(16); - fileConfig.entrySet().forEach(x -> optionMap.put(x.getKey(),String.valueOf(x.getValue()))); + Map optionMap = new HashMap<>(16); + fileConfig.entrySet().forEach(x -> optionMap.put(x.getKey(), String.valueOf(x.getValue()))); writer.options(optionMap); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java index 7aba1de244..7fd3961868 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java @@ -37,7 +37,7 @@ public class HdfsFileWriter extends BaseFileWriter { @Override public void write(Dataset data, SparkRuntimeEnvironment environment) { - outputImpl(data,"hdfs://"); + outputImpl(data, "hdfs://"); } @Override diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java index a35db3cfbe..1741ff20fb 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java @@ -37,7 +37,7 @@ public class LocalFileWriter extends BaseFileWriter { @Override public void write(Dataset data, SparkRuntimeEnvironment environment) { - outputImpl(data,"file://"); + outputImpl(data, "file://"); } @Override diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java index e35eeaa886..f94d103508 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java @@ -48,10 +48,10 @@ public class JsonUtils { private static final ObjectMapper MAPPER = new ObjectMapper() .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) - .configure(ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,true) + .configure(ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true) .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) .configure(REQUIRE_SETTERS_FOR_GETTERS, true) - .configure(FAIL_ON_EMPTY_BEANS,false) + .configure(FAIL_ON_EMPTY_BEANS, false) .setTimeZone(TimeZone.getDefault()) .setDateFormat(new SimpleDateFormat(Constants.YYYY_MM_DD_HH_MM_SS)); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index cd564e2a2d..f5f966bba4 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-datasource-plugin org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-datasource-all diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java index 50de82f93d..ac3f96d46c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java @@ -89,14 +89,15 @@ public class CommonDataSourceClient implements DataSourceClient { @Override public void checkClient() { - //Checking data source client + // Checking data source client Stopwatch stopwatch = Stopwatch.createStarted(); try { this.jdbcTemplate.execute(this.baseConnectionParam.getValidationQuery()); } catch (Exception e) { throw new RuntimeException("JDBC connect failed", e); } finally { - logger.info("Time to execute check jdbc client with sql {} for {} ms ", this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + logger.info("Time to execute check jdbc client with sql {} for {} ms ", + this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java index 728a91956d..fd14837dad 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.api.datasource; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class BaseHDFSConnectionParam extends BaseConnectionParam { + protected String principal; protected String javaSecurityKrb5Conf; protected String loginUserKeytabUsername; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java index 3cfac25663..e30a638125 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java @@ -34,7 +34,6 @@ public interface DataSourceProcessor { */ BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson); - /** * check datasource param is valid */ diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java index 52b76d2c41..6425064b3d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java @@ -17,19 +17,21 @@ package org.apache.dolphinscheduler.plugin.datasource.api.plugin; +import static java.lang.String.format; + import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static java.lang.String.format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataSourcePluginManager { + private static final Logger logger = LoggerFactory.getLogger(DataSourcePluginManager.class); private final Map datasourceClientMap = new ConcurrentHashMap<>(); @@ -40,7 +42,8 @@ public class DataSourcePluginManager { public void installPlugin() { - PrioritySPIFactory prioritySPIFactory = new PrioritySPIFactory<>(DataSourceChannelFactory.class); + PrioritySPIFactory prioritySPIFactory = + new PrioritySPIFactory<>(DataSourceChannelFactory.class); for (Map.Entry entry : prioritySPIFactory.getSPIMap().entrySet()) { final DataSourceChannelFactory factory = entry.getValue(); final String name = entry.getKey(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java index b694f4534f..dbdbe3bf68 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DataSourceProcessorManager { + private static final Logger logger = LoggerFactory.getLogger(DataSourceProcessorManager.class); private static final Map dataSourceProcessorMap = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java index 7cdd5c98d1..0b4994116a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java @@ -18,13 +18,14 @@ package org.apache.dolphinscheduler.plugin.datasource.api.plugin; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataSourceProcessorProvider { + private static final Logger logger = LoggerFactory.getLogger(DataSourceProcessorProvider.class); private DataSourceProcessorManager dataSourcePluginManager; @@ -34,6 +35,7 @@ public class DataSourceProcessorProvider { } private static class DataSourceClientProviderHolder { + private static final DataSourceProcessorProvider INSTANCE = new DataSourceProcessorProvider(); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java index 73af3517db..9777272508 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java @@ -17,7 +17,15 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATA_QUALITY_JAR_NAME; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF_PATH; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.KERBEROS; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_USER_KEY_TAB_PATH; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_USER_KEY_TAB_USERNAME; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESOURCE_UPLOAD_PATH; import static org.apache.dolphinscheduler.spi.utils.Constants.RESOURCE_STORAGE_TYPE; import org.apache.dolphinscheduler.common.enums.ResUploadType; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java index 80c2d68cc8..7d95f60658 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java @@ -17,19 +17,21 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; -import com.fasterxml.jackson.databind.JsonNode; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.Connection; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; + public class DataSourceUtils { public DataSourceUtils() { @@ -83,7 +85,8 @@ public class DataSourceUtils { } public static DataSourceProcessor getDatasourceProcessor(DbType dbType) { - Map dataSourceProcessorMap = DataSourceProcessorProvider.getInstance().getDataSourceProcessorMap(); + Map dataSourceProcessorMap = + DataSourceProcessorProvider.getInstance().getDataSourceProcessorMap(); if (!dataSourceProcessorMap.containsKey(dbType.name())) { throw new IllegalArgumentException("illegal datasource type"); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java index 05c9840795..edfebe4fe6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; * Used for test. */ public class MySQLConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "MySQLConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml index 0ca15dc16d..932cc2af29 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml @@ -15,16 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 - dolphinscheduler-datasource-plugin org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-datasource-athena jar @@ -41,4 +39,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java index 216c0112ac..a83c6b9fb7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java @@ -23,8 +23,9 @@ import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; import org.apache.dolphinscheduler.spi.enums.DbType; public class AthenaDataSourceChannel implements DataSourceChannel { + @Override public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { - return new AthenaDataSourceClient(baseConnectionParam,dbType); + return new AthenaDataSourceClient(baseConnectionParam, dbType); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java index d1439570c3..1b2ed367d0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class AthenaDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public DataSourceChannel create() { return new AthenaDataSourceChannel(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java index 58bde0cce8..20755d2f30 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; public class AthenaDataSourceClient extends CommonDataSourceClient { + public AthenaDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { super(baseConnectionParam, dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java index ea89de7a3c..b52b5cc362 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java @@ -26,6 +26,7 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) public class AthenaDataSourceParamDTO extends BaseDataSourceParamDTO { + protected String awsRegion; @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java index 4187b37a50..402e34808f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.datasource.athena.param; -import com.google.auto.service.AutoService; -import org.apache.commons.collections4.MapUtils; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; @@ -30,6 +28,8 @@ import org.apache.dolphinscheduler.spi.utils.Constants; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; +import org.apache.commons.collections4.MapUtils; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -38,6 +38,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import com.google.auto.service.AutoService; + @AutoService(DataSourceProcessor.class) public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { @@ -48,11 +50,9 @@ public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - AthenaConnectionParam - connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson); + AthenaConnectionParam connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson); - AthenaDataSourceParamDTO - athenaDatasourceParamDTO = new AthenaDataSourceParamDTO(); + AthenaDataSourceParamDTO athenaDatasourceParamDTO = new AthenaDataSourceParamDTO(); athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion()); athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); athenaDatasourceParamDTO.setUserName(connectionParams.getUser()); @@ -66,8 +66,7 @@ public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { AthenaDataSourceParamDTO athenaParam = (AthenaDataSourceParamDTO) datasourceParam; String address = String.format("%s%s=%s;", Constants.JDBC_ATHENA, "AwsRegion", athenaParam.getAwsRegion()); - AthenaConnectionParam - athenaConnectionParam = new AthenaConnectionParam(); + AthenaConnectionParam athenaConnectionParam = new AthenaConnectionParam(); athenaConnectionParam.setUser(athenaParam.getUserName()); athenaConnectionParam.setPassword(PasswordUtils.encodePassword(athenaParam.getPassword())); athenaConnectionParam.setAwsRegion(athenaParam.getAwsRegion()); @@ -99,8 +98,7 @@ public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - AthenaConnectionParam - athenaConnectionParam = (AthenaConnectionParam) connectionParam; + AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam; if (!StringUtils.isEmpty(athenaConnectionParam.getOther())) { return String.format("%s%s", athenaConnectionParam.getJdbcUrl(), athenaConnectionParam.getOther()); } @@ -112,7 +110,7 @@ public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam; Class.forName(this.getDatasourceDriver()); return DriverManager.getConnection(this.getJdbcUrl(connectionParam), - athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword())); + athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java index 03534e7312..d756226522 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class ClickHouseDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "clickhouse"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java index 4cf56ffffb..df31fae575 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.clickhouse.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class ClickHouseConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "ClickHouseConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java index 7bbc5ba4c5..5310c7bae6 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java @@ -49,8 +49,7 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor { public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { ClickHouseConnectionParam connectionParams = (ClickHouseConnectionParam) createConnectionParams(connectionJson); - ClickHouseDataSourceParamDTO - clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO(); + ClickHouseDataSourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO(); clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); @@ -66,7 +65,8 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor { @Override public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { ClickHouseDataSourceParamDTO clickHouseParam = (ClickHouseDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); + String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), + clickHouseParam.getPort()); String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); ClickHouseConnectionParam clickhouseConnectionParam = new ClickHouseConnectionParam(); @@ -112,7 +112,8 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor { ClickHouseConnectionParam clickhouseConnectionParam = (ClickHouseConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), - clickhouseConnectionParam.getUser(), PasswordUtils.decodePassword(clickhouseConnectionParam.getPassword())); + clickhouseConnectionParam.getUser(), + PasswordUtils.decodePassword(clickhouseConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java index 88be896c25..a0a1667690 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.db2.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class Db2ConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "Db2ConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java index e8272e0c10..eaf07332fd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java @@ -50,8 +50,7 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor { public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); - Db2DataSourceParamDTO - db2DatasourceParamDTO = new Db2DataSourceParamDTO(); + Db2DataSourceParamDTO db2DatasourceParamDTO = new Db2DataSourceParamDTO(); db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java index 12556b66be..96ee007c8d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class HiveDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "hive"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java index ac432df4d6..a403ff2b27 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.param; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSConnectionParam; public class HiveConnectionParam extends BaseHDFSConnectionParam { + @Override public String toString() { return "HiveConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java index ab721c9774..1cbc0f4939 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java @@ -50,8 +50,7 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - HiveDataSourceParamDTO - hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); + HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java index 64788c0111..e57fc7e61d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class MySQLDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "mysql"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java index 17e3152592..89137c0016 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.mysql.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class MySQLConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "MySQLConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java index 5d13cec7f5..3da0072f2f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java @@ -55,7 +55,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; - private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; + private static final String APPEND_PARAMS = + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; @Override public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @@ -64,10 +65,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - MySQLConnectionParam - connectionParams = (MySQLConnectionParam) createConnectionParams(connectionJson); - MySQLDataSourceParamDTO - mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO(); + MySQLConnectionParam connectionParams = (MySQLConnectionParam) createConnectionParams(connectionJson); + MySQLDataSourceParamDTO mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO(); mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); @@ -85,11 +84,11 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { MySQLDataSourceParamDTO mysqlDatasourceParam = (MySQLDataSourceParamDTO) dataSourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); + String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), + mysqlDatasourceParam.getPort()); String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); - MySQLConnectionParam - mysqlConnectionParam = new MySQLConnectionParam(); + MySQLConnectionParam mysqlConnectionParam = new MySQLConnectionParam(); mysqlConnectionParam.setJdbcUrl(jdbcUrl); mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); mysqlConnectionParam.setAddress(address); @@ -120,8 +119,7 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - MySQLConnectionParam - mysqlConnectionParam = (MySQLConnectionParam) connectionParam; + MySQLConnectionParam mysqlConnectionParam = (MySQLConnectionParam) connectionParam; String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); if (!StringUtils.isEmpty(mysqlConnectionParam.getOther())) { return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java index 8fcc47766b..dedbce4946 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class OracleDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "oracle"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java index 9bbac59e15..9babd1ed21 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbConnectType; public class OracleConnectionParam extends BaseConnectionParam { + protected DbConnectType connectType; public DbConnectType getConnectType() { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java index 4572a09af4..8aa6e566b7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class PostgreSQLDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "postgresql"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java index 94353a772f..d39681344c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.postgresql.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class PostgreSQLConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "PostgreSQLConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java index 79db1dff54..373e5a092a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java @@ -49,8 +49,7 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { PostgreSQLConnectionParam connectionParams = (PostgreSQLConnectionParam) createConnectionParams(connectionJson); - PostgreSQLDataSourceParamDTO - postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO(); + PostgreSQLDataSourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO(); postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); @@ -67,7 +66,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { PostgreSQLDataSourceParamDTO postgreSqlParam = (PostgreSQLDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); + String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), + postgreSqlParam.getPort()); String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); PostgreSQLConnectionParam postgreSqlConnectionParam = new PostgreSQLConnectionParam(); @@ -113,7 +113,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor { PostgreSQLConnectionParam postgreSqlConnectionParam = (PostgreSQLConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), - postgreSqlConnectionParam.getUser(), PasswordUtils.decodePassword(postgreSqlConnectionParam.getPassword())); + postgreSqlConnectionParam.getUser(), + PasswordUtils.decodePassword(postgreSqlConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java index 95d0f25b8f..ed1292ffc9 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class PrestoDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "presto"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java index 8b590ce8bb..eba0cba306 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.presto.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class PrestoConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "PrestoConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java index 79b5c3cc52..4779af9ce0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java @@ -55,8 +55,7 @@ public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor { String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); - PrestoDataSourceParamDTO - prestoDatasourceParamDTO = new PrestoDataSourceParamDTO(); + PrestoDataSourceParamDTO prestoDatasourceParamDTO = new PrestoDataSourceParamDTO(); prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml index 0a290f31f5..041dbea12e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-datasource-plugin org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-datasource-redshift jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java index 13e8172478..9992f51199 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java @@ -23,8 +23,9 @@ import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; import org.apache.dolphinscheduler.spi.enums.DbType; public class RedshiftDataSourceChannel implements DataSourceChannel { + @Override public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { - return new RedshiftDataSourceClient(baseConnectionParam,dbType); + return new RedshiftDataSourceClient(baseConnectionParam, dbType); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java index 576a519a2e..25a587ae06 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class RedshiftDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public DataSourceChannel create() { return new RedshiftDataSourceChannel(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java index f10f7e1f80..c9a31080d5 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; public class RedshiftDataSourceClient extends CommonDataSourceClient { + public RedshiftDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { super(baseConnectionParam, dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java index 6022d62b47..dd3c2d1b6e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.redshift.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class RedshiftConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "RedshiftConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java index c8586a275c..88cebe4dc2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java @@ -50,14 +50,12 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - RedshiftConnectionParam - connectionParams = (RedshiftConnectionParam) createConnectionParams(connectionJson); + RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) createConnectionParams(connectionJson); String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); - RedshiftDataSourceParamDTO - redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); + RedshiftDataSourceParamDTO redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); redshiftDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); redshiftDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); @@ -70,11 +68,11 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { RedshiftDataSourceParamDTO redshiftParam = (RedshiftDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_REDSHIFT, redshiftParam.getHost(), redshiftParam.getPort()); + String address = + String.format("%s%s:%s", Constants.JDBC_REDSHIFT, redshiftParam.getHost(), redshiftParam.getPort()); String jdbcUrl = address + Constants.SLASH + redshiftParam.getDatabase(); - RedshiftConnectionParam - redshiftConnectionParam = new RedshiftConnectionParam(); + RedshiftConnectionParam redshiftConnectionParam = new RedshiftConnectionParam(); redshiftConnectionParam.setUser(redshiftParam.getUserName()); redshiftConnectionParam.setPassword(PasswordUtils.encodePassword(redshiftParam.getPassword())); redshiftConnectionParam.setOther(transformOther(redshiftParam.getOther())); @@ -105,8 +103,7 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - RedshiftConnectionParam - redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; + RedshiftConnectionParam redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; if (!StringUtils.isEmpty(redshiftConnectionParam.getOther())) { return String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(), redshiftConnectionParam.getOther()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java index f49c0b3fd9..dbda3da5bd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class SparkDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "spark"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java index 2dbeaa7a19..b33288d64b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.spark.param; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSConnectionParam; public class SparkConnectionParam extends BaseHDFSConnectionParam { + @Override public String toString() { return "SparkConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java index f5dfb2e5ca..b572ebe86a 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java @@ -53,11 +53,9 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SparkConnectionParam - connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); + SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); - SparkDataSourceParamDTO - sparkDatasourceParamDTO = new SparkDataSourceParamDTO(); + SparkDataSourceParamDTO sparkDatasourceParamDTO = new SparkDataSourceParamDTO(); sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); @@ -68,7 +66,8 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor { StringBuilder hosts = new StringBuilder(); String[] tmpArray = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA); - Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA)); + Arrays.stream(hostPortArray) + .forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA)); hosts.deleteCharAt(hosts.length() - 1); sparkDatasourceParamDTO.setHost(hosts.toString()); @@ -89,8 +88,7 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor { String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); - SparkConnectionParam - sparkConnectionParam = new SparkConnectionParam(); + SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); sparkConnectionParam.setPassword(PasswordUtils.encodePassword(sparkDatasourceParam.getPassword())); sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); @@ -128,8 +126,7 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - SparkConnectionParam - sparkConnectionParam = (SparkConnectionParam) connectionParam; + SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; if (!StringUtils.isEmpty(sparkConnectionParam.getOther())) { return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java index 58a0a7468b..e76f520d1e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java @@ -24,6 +24,7 @@ import com.google.auto.service.AutoService; @AutoService(DataSourceChannelFactory.class) public class SQLServerDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "sqlserver"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java index c6771cd091..777d6d3da8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class SQLServerConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "SQLServerConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java index ee874979ae..fbe07da938 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java @@ -64,7 +64,8 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { SQLServerDataSourceParamDTO sqlServerParam = (SQLServerDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); + String address = + String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); SQLServerConnectionParam sqlServerConnectionParam = new SQLServerConnectionParam(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6419a97365..b334865e88 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -29,6 +29,9 @@ import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread; import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.task.TaskPluginManager; + +import javax.annotation.PostConstruct; + import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,8 +42,6 @@ import org.springframework.cache.annotation.EnableCaching; import org.springframework.context.annotation.ComponentScan; import org.springframework.transaction.annotation.EnableTransactionManagement; -import javax.annotation.PostConstruct; - @SpringBootApplication @ComponentScan("org.apache.dolphinscheduler") @EnableTransactionManagement diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java index 262a5acc62..6c37353ebd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java @@ -33,11 +33,12 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * TaskExecutionContext builder */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java index bf5a08b477..1352d125c3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.cache; -import lombok.NonNull; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import java.util.Collection; @@ -68,4 +67,4 @@ public interface ProcessInstanceExecCacheManager { Collection getAll(); void clearCache(); -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java index 681d5129bc..e4be3e8397 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.master.cache; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import java.util.Collection; @@ -49,4 +48,4 @@ public interface StreamTaskInstanceExecCacheManager { * @return all streamTaskExecuteRunnable in cache */ Collection getAll(); -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java index 3588e3d336..105aa39708 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java @@ -26,12 +26,12 @@ import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.springframework.stereotype.Component; import com.google.common.collect.ImmutableList; -import lombok.NonNull; - /** * cache of process instance id and WorkflowExecuteThread */ @@ -75,4 +75,4 @@ public class ProcessInstanceExecCacheManagerImpl implements ProcessInstanceExecC public void clearCache() { processInstanceExecMaps.clear(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java index 1033ab35e5..2d4e46b7c6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java @@ -18,29 +18,28 @@ package org.apache.dolphinscheduler.server.master.cache.impl; import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; -import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import java.util.Collection; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.springframework.stereotype.Component; import com.google.common.collect.ImmutableList; -import lombok.NonNull; - /** * cache of process instance id and WorkflowExecuteThread */ @Component public class StreamTaskInstanceExecCacheManagerImpl implements StreamTaskInstanceExecCacheManager { - private final ConcurrentHashMap streamTaskInstanceExecMaps = new ConcurrentHashMap<>(); + private final ConcurrentHashMap streamTaskInstanceExecMaps = + new ConcurrentHashMap<>(); @PostConstruct public void registerMetrics() { @@ -71,4 +70,4 @@ public class StreamTaskInstanceExecCacheManagerImpl implements StreamTaskInstanc public Collection getAll() { return ImmutableList.copyOf(streamTaskInstanceExecMaps.values()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index b0426f477a..f3b3d78fb7 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -17,12 +17,18 @@ package org.apache.dolphinscheduler.server.master.config; -import lombok.Data; +import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS; + import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelector; import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; + +import java.time.Duration; + +import lombok.Data; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -31,10 +37,6 @@ import org.springframework.validation.Errors; import org.springframework.validation.Validator; import org.springframework.validation.annotation.Validated; -import java.time.Duration; - -import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS; - @Data @Validated @Configuration diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index f073fd55f3..5d31f82fe9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.consumer; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; @@ -40,12 +39,9 @@ import org.apache.dolphinscheduler.service.exceptions.TaskPriorityQueueException import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.TaskPriority; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; +import org.apache.commons.collections.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -55,6 +51,13 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + /** * TaskUpdateQueue consumer */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java index 793dc40439..28f26d5cb8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java @@ -45,7 +45,8 @@ public class WorkflowExecuteController { @GetMapping("") @ResponseStatus(HttpStatus.OK) public WorkflowExecuteDto queryExecuteData(@RequestParam("id") int processInstanceId) { - Optional workflowExecuteDtoOptional = executingService.queryWorkflowExecutingData(processInstanceId); + Optional workflowExecuteDtoOptional = + executingService.queryWorkflowExecutingData(processInstanceId); return workflowExecuteDtoOptional.orElse(null); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java index d439c28267..3b1a0d4839 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java @@ -85,7 +85,7 @@ public class ExecutorDispatcher implements InitializingBean { Host host = hostManager.select(context); if (StringUtils.isEmpty(host.getAddress())) { logger.warn("fail to execute : {} due to no suitable worker, current task needs worker group {} to execute", - context.getCommand(), context.getWorkerGroup()); + context.getCommand(), context.getWorkerGroup()); return false; } context.setHost(host); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java index 62da82156f..93541a9b37 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java @@ -27,8 +27,6 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; -import org.apache.dolphinscheduler.server.master.processor.TaskExecuteResponseProcessor; -import org.apache.dolphinscheduler.server.master.processor.TaskExecuteRunningProcessor; import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor; import org.apache.dolphinscheduler.server.master.processor.TaskRecallProcessor; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java index bbf84a3d74..fd7dc97e3e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; @@ -26,14 +25,14 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; +import org.springframework.beans.factory.annotation.Autowired; + /** * common host manager */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java index 3ad46ddd53..0ac1a53180 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; import org.apache.dolphinscheduler.remote.utils.Host; @@ -26,10 +25,9 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWeight import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.LowerWeightRoundRobin; import org.apache.dolphinscheduler.server.master.registry.WorkerInfoChangeListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.annotation.PostConstruct; +import org.apache.commons.collections.CollectionUtils; + import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -40,6 +38,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * lower weight host manager */ @@ -90,8 +93,8 @@ public class LowerWeightHostManager extends CommonHostManager { throw new UnsupportedOperationException("not support"); } - private class WorkerWeightListener implements WorkerInfoChangeListener { + @Override public void notify(Map> workerGroups, Map workerNodeInfo) { syncWorkerResources(workerGroups, workerNodeInfo); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java index 0e61599e79..693ff856e5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java @@ -24,7 +24,8 @@ import java.util.Collection; /** * AbstractSelector */ -public abstract class AbstractSelector implements Selector { +public abstract class AbstractSelector implements Selector { + @Override public T select(Collection source) { @@ -36,7 +37,7 @@ public abstract class AbstractSelector implements Selector { * if only one , return directly */ if (source.size() == 1) { - return (T)source.toArray()[0]; + return (T) source.toArray()[0]; } return doSelect(source); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java index a441582235..b3cb2ae75c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java @@ -39,7 +39,8 @@ public class HostWeight { private final int waitingTaskCount; - public HostWeight(HostWorker hostWorker, double cpu, double memory, double loadAverage, int waitingTaskCount, long startTime) { + public HostWeight(HostWorker hostWorker, double cpu, double memory, double loadAverage, int waitingTaskCount, + long startTime) { this.hostWorker = hostWorker; this.weight = calculateWeight(cpu, memory, loadAverage, startTime); this.currentWeight = this.weight; @@ -63,7 +64,7 @@ public class HostWeight { } public Host getHost() { - return (Host)hostWorker; + return (Host) hostWorker; } public int getWaitingTaskCount() { @@ -73,11 +74,11 @@ public class HostWeight { @Override public String toString() { return "HostWeight{" - + "hostWorker=" + hostWorker - + ", weight=" + weight - + ", currentWeight=" + currentWeight - + ", waitingTaskCount=" + waitingTaskCount - + '}'; + + "hostWorker=" + hostWorker + + ", weight=" + weight + + ", currentWeight=" + currentWeight + + ", waitingTaskCount=" + waitingTaskCount + + '}'; } private double calculateWeight(double cpu, double memory, double loadAverage, long startTime) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java index 3388129b2a..8ddfff6da5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java @@ -62,20 +62,20 @@ public class LowerWeightRoundRobin extends AbstractSelector { if (CollectionUtils.isEmpty(sources)) { return Collections.emptyList(); } - List zeroWaitingTask = sources.stream().filter(h -> h.getWaitingTaskCount() == 0).collect(Collectors.toList()); + List zeroWaitingTask = + sources.stream().filter(h -> h.getWaitingTaskCount() == 0).collect(Collectors.toList()); if (!zeroWaitingTask.isEmpty()) { return zeroWaitingTask; } HostWeight hostWeight = sources.stream().min(Comparator.comparing(HostWeight::getWaitingTaskCount)).get(); List waitingTask = Lists.newArrayList(hostWeight); - List equalWaitingTask = sources.stream().filter(h -> !h.getHost().equals(hostWeight.getHost()) && h.getWaitingTaskCount() == hostWeight.getWaitingTaskCount()) - .collect(Collectors.toList()); + List equalWaitingTask = sources.stream() + .filter(h -> !h.getHost().equals(hostWeight.getHost()) + && h.getWaitingTaskCount() == hostWeight.getWaitingTaskCount()) + .collect(Collectors.toList()); if (!equalWaitingTask.isEmpty()) { waitingTask.addAll(equalWaitingTask); } return waitingTask; } } - - - diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java index 5639fc4ce2..8f21acef6d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java @@ -33,13 +33,15 @@ import org.springframework.stereotype.Service; @Service public class RoundRobinSelector extends AbstractSelector { - private ConcurrentMap> workGroupWeightMap = new ConcurrentHashMap<>(); + private ConcurrentMap> workGroupWeightMap = + new ConcurrentHashMap<>(); private static final int RECYCLE_PERIOD = 100000; private AtomicBoolean updateLock = new AtomicBoolean(); protected static class WeightedRoundRobin { + private int weight; private AtomicLong current = new AtomicLong(0); private long lastUpdate; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java index f3a9a6579e..845c77e296 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.server.master.event; -import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.StateEventType; -import io.netty.channel.Channel; - import javax.annotation.Nullable; +import lombok.NonNull; +import io.netty.channel.Channel; + /** * state event */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java index 00808b2e29..93d4a7d4b3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java @@ -29,8 +29,8 @@ public interface StateEventHandler { * @throws StateEventHandleException this exception means it can be recovered. * @throws StateEventHandleError this exception means it cannot be recovered, so the event need to drop. */ - boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) - throws StateEventHandleException, StateEventHandleError; + boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, + StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError; StateEventType getEventType(); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java index b436b55890..b91da8cc51 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java @@ -30,8 +30,8 @@ public class StateEventHandlerManager { static { ServiceLoader.load(StateEventHandler.class) - .forEach(stateEventHandler -> stateEventHandlerMap.put(stateEventHandler.getEventType(), - stateEventHandler)); + .forEach(stateEventHandler -> stateEventHandlerMap.put(stateEventHandler.getEventType(), + stateEventHandler)); } public static Optional getStateEventHandler(StateEventType stateEventType) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java index 1d5f7a3883..4a55b6b430 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java @@ -17,15 +17,17 @@ package org.apache.dolphinscheduler.server.master.event; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; +import com.google.auto.service.AutoService; @AutoService(StateEventHandler.class) public class TaskRetryStateEventHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java index fe449a9f6f..3dc86b3a37 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java @@ -27,13 +27,14 @@ import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.Optional; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Optional; - @Component public class TaskRunningEventHandler implements TaskEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java index 1ad96b9346..157d62e863 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java @@ -17,14 +17,15 @@ package org.apache.dolphinscheduler.server.master.event; -import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.enums.StateEventType; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; -import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import io.netty.channel.Channel; @Data @Builder diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java index 9efe773b19..034bf8107f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.event; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -28,9 +27,12 @@ import org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessor; import org.apache.dolphinscheduler.server.master.runner.task.TaskAction; import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class TaskTimeoutStateEventHandler implements StateEventHandler { @@ -38,17 +40,17 @@ public class TaskTimeoutStateEventHandler implements StateEventHandler { @Override public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, - StateEvent stateEvent) throws StateEventHandleError { + StateEvent stateEvent) throws StateEventHandleError { TaskStateEvent taskStateEvent = (TaskStateEvent) stateEvent; TaskMetrics.incTaskInstanceByState("timeout"); workflowExecuteRunnable.checkTaskInstanceByStateEvent(taskStateEvent); TaskInstance taskInstance = - workflowExecuteRunnable.getTaskInstance(taskStateEvent.getTaskInstanceId()).orElseThrow( - () -> new StateEventHandleError(String.format( - "Cannot find the task instance from workflow execute runnable, taskInstanceId: %s", - taskStateEvent.getTaskInstanceId()))); + workflowExecuteRunnable.getTaskInstance(taskStateEvent.getTaskInstanceId()).orElseThrow( + () -> new StateEventHandleError(String.format( + "Cannot find the task instance from workflow execute runnable, taskInstanceId: %s", + taskStateEvent.getTaskInstanceId()))); logger.info("Handle task instance state timout event, taskInstanceId: {}", taskStateEvent.getTaskInstanceId()); @@ -56,22 +58,22 @@ public class TaskTimeoutStateEventHandler implements StateEventHandler { return true; } TaskTimeoutStrategy taskTimeoutStrategy = taskInstance.getTaskDefine() - .getTimeoutNotifyStrategy(); + .getTimeoutNotifyStrategy(); Map activeTaskProcessMap = workflowExecuteRunnable - .getActiveTaskProcessMap(); + .getActiveTaskProcessMap(); if ((TaskTimeoutStrategy.FAILED == taskTimeoutStrategy - || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy)) { + || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy)) { if (activeTaskProcessMap.containsKey(taskInstance.getTaskCode())) { ITaskProcessor taskProcessor = activeTaskProcessMap.get(taskInstance.getTaskCode()); taskProcessor.action(TaskAction.TIMEOUT); } else { logger.warn( - "cannot find the task processor for task {}, so skip task processor action.", - taskInstance.getTaskCode()); + "cannot find the task processor for task {}, so skip task processor action.", + taskInstance.getTaskCode()); } } if (TaskTimeoutStrategy.WARN == taskTimeoutStrategy - || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) { + || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) { workflowExecuteRunnable.processTimeout(); workflowExecuteRunnable.taskTimeout(taskInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java index 8dcca91d89..02d08a6616 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java @@ -20,10 +20,11 @@ package org.apache.dolphinscheduler.server.master.event; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; -import com.google.auto.service.AutoService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class TaskWaitTaskGroupStateHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java index cac30a8072..5c9f901e68 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java @@ -36,14 +36,14 @@ public class WorkflowBlockStateEventHandler implements StateEventHandler { private static final Logger logger = LoggerFactory.getLogger(WorkflowBlockStateEventHandler.class); @Override - public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) - throws StateEventHandleError { + public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, + StateEvent stateEvent) throws StateEventHandleError { logger.info("Handle workflow instance state block event"); Optional taskInstanceOptional = - workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId()); + workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId()); if (!taskInstanceOptional.isPresent()) { throw new StateEventHandleError("Cannot find taskInstance from taskMap by taskInstanceId: " - + stateEvent.getTaskInstanceId()); + + stateEvent.getTaskInstanceId()); } TaskInstance task = taskInstanceOptional.get(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java index 8c5e27c117..13d82f0671 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java @@ -53,27 +53,27 @@ public class WorkflowStartEventHandler implements WorkflowEventHandler { public void handleWorkflowEvent(final WorkflowEvent workflowEvent) throws WorkflowEventHandleError { logger.info("Handle workflow start event, begin to start a workflow, event: {}", workflowEvent); WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId( - workflowEvent.getWorkflowInstanceId()); + workflowEvent.getWorkflowInstanceId()); if (workflowExecuteRunnable == null) { throw new WorkflowEventHandleError( - "The workflow start event is invalid, cannot find the workflow instance from cache"); + "The workflow start event is invalid, cannot find the workflow instance from cache"); } ProcessInstanceMetrics.incProcessInstanceByState("submit"); ProcessInstance processInstance = workflowExecuteRunnable.getProcessInstance(); CompletableFuture.supplyAsync(workflowExecuteRunnable::call, workflowExecuteThreadPool) - .thenAccept(workflowSubmitStatue -> { - if (WorkflowSubmitStatue.SUCCESS == workflowSubmitStatue) { - // submit failed will resend the event to workflow event queue - logger.info("Success submit the workflow instance"); - if (processInstance.getTimeout() > 0) { - stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance); + .thenAccept(workflowSubmitStatue -> { + if (WorkflowSubmitStatue.SUCCESS == workflowSubmitStatue) { + // submit failed will resend the event to workflow event queue + logger.info("Success submit the workflow instance"); + if (processInstance.getTimeout() > 0) { + stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance); + } + } else { + logger.error("Failed to submit the workflow instance, will resend the workflow start event: {}", + workflowEvent); + workflowEventQueue.addEvent(workflowEvent); } - } else { - logger.error("Failed to submit the workflow instance, will resend the workflow start event: {}", - workflowEvent); - workflowEventQueue.addEvent(workflowEvent); - } - }); + }); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java index 3267108228..cf65575caa 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java @@ -17,11 +17,16 @@ package org.apache.dolphinscheduler.server.master.event; -import io.netty.channel.Channel; -import lombok.*; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import io.netty.channel.Channel; + @Data @Builder @AllArgsConstructor diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java index c3f49111a7..65015f0458 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java @@ -17,15 +17,17 @@ package org.apache.dolphinscheduler.server.master.event; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class WorkflowStateEventHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java index 4a9fd99392..d68e97c5e3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java @@ -21,10 +21,11 @@ import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; -import com.google.auto.service.AutoService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class WorkflowTimeoutStateEventHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java index 0ed522e7cd..a0d153ab62 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.server.master.metrics; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; @UtilityClass public class MasterServerMetrics { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java index 867f8ae548..d610778424 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java @@ -23,13 +23,14 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; + import com.google.common.collect.ImmutableSet; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; -import lombok.experimental.UtilityClass; @UtilityClass public class ProcessInstanceMetrics { @@ -46,21 +47,20 @@ public class ProcessInstanceMetrics { Counter.builder("ds.workflow.instance.count") .tag("state", state) .description(String.format("Process instance %s total count", state)) - .register(Metrics.globalRegistry) - ); + .register(Metrics.globalRegistry)); } } private final Timer commandQueryTimer = - Timer.builder("ds.workflow.command.query.duration") - .description("Command query duration") - .register(Metrics.globalRegistry); + Timer.builder("ds.workflow.command.query.duration") + .description("Command query duration") + .register(Metrics.globalRegistry); private final Timer processInstanceGenerateTimer = - Timer.builder("ds.workflow.instance.generate.duration") - .description("Process instance generated duration") - .register(Metrics.globalRegistry); + Timer.builder("ds.workflow.instance.generate.duration") + .description("Process instance generated duration") + .register(Metrics.globalRegistry); public void recordCommandQueryTime(long milliseconds) { commandQueryTimer.record(milliseconds, TimeUnit.MILLISECONDS); @@ -72,14 +72,14 @@ public class ProcessInstanceMetrics { public synchronized void registerProcessInstanceRunningGauge(Supplier function) { Gauge.builder("ds.workflow.instance.running", function) - .description("The current running process instance count") - .register(Metrics.globalRegistry); + .description("The current running process instance count") + .register(Metrics.globalRegistry); } public synchronized void registerProcessInstanceResubmitGauge(Supplier function) { Gauge.builder("ds.workflow.instance.resubmit", function) - .description("The current process instance need to resubmit count") - .register(Metrics.globalRegistry); + .description("The current process instance need to resubmit count") + .register(Metrics.globalRegistry); } public void incProcessInstanceByState(final String state) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java index e1b1307b80..9b901d7b60 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java @@ -17,17 +17,19 @@ package org.apache.dolphinscheduler.server.master.metrics; -import com.facebook.presto.jdbc.internal.guava.collect.ImmutableSet; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; - import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; + +import com.facebook.presto.jdbc.internal.guava.collect.ImmutableSet; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Metrics; + @UtilityClass public class TaskMetrics { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java index f9b9a119dc..39c4776ce1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java @@ -32,7 +32,6 @@ import org.springframework.cache.CacheManager; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -48,7 +47,8 @@ public class CacheProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.CACHE_EXPIRE == command.getType(), String.format("invalid command type: %s", command.getType())); + Preconditions.checkArgument(CommandType.CACHE_EXPIRE == command.getType(), + String.format("invalid command type: %s", command.getType())); CacheExpireCommand cacheExpireCommand = JSONUtils.parseObject(command.getBody(), CacheExpireCommand.class); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java index 322870b2c1..c96d4fec06 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java @@ -26,7 +26,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; public class HostUpdateResponseProcessor implements NettyRequestProcessor { @@ -35,9 +34,11 @@ public class HostUpdateResponseProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); + Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_RESPONSE == command.getType(), + String.format("invalid command type : %s", command.getType())); - HostUpdateResponseProcessor responseCommand = JSONUtils.parseObject(command.getBody(), HostUpdateResponseProcessor.class); + HostUpdateResponseProcessor responseCommand = + JSONUtils.parseObject(command.getBody(), HostUpdateResponseProcessor.class); logger.info("received process host response command : {}", responseCommand); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java index a54fc2cbac..c9d9b6fe8e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -31,11 +29,15 @@ import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + /** * handle state event received from master/api */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java index ce31b1a018..43822fce58 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; @@ -28,11 +26,15 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + /** * handle state event received from master/api */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java index 8f66b612db..378562609c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java @@ -25,13 +25,13 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -55,16 +55,16 @@ public class TaskExecuteResponseProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESULT == command.getType(), - String.format("invalid command type : %s", command.getType())); + String.format("invalid command type : %s", command.getType())); TaskExecuteResultCommand taskExecuteResultMessage = JSONUtils.parseObject(command.getBody(), - TaskExecuteResultCommand.class); + TaskExecuteResultCommand.class); TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, - channel, - taskExecuteResultMessage.getMessageSenderAddress()); + channel, + taskExecuteResultMessage.getMessageSenderAddress()); try { LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskResultEvent.getProcessInstanceId(), - taskResultEvent.getTaskInstanceId()); + taskResultEvent.getTaskInstanceId()); logger.info("Received task execute result, event: {}", taskResultEvent); taskEventService.addEvent(taskResultEvent); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java index 47a66a71f0..2a0d265033 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java @@ -31,7 +31,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -53,13 +52,15 @@ public class TaskExecuteRunningProcessor implements NettyRequestProcessor { */ @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteRunningCommand taskExecuteRunningMessage = JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class); + Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING == command.getType(), + String.format("invalid command type : %s", command.getType())); + TaskExecuteRunningCommand taskExecuteRunningMessage = + JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class); logger.info("taskExecuteRunningCommand: {}", taskExecuteRunningMessage); TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, - channel, - taskExecuteRunningMessage.getMessageSenderAddress()); + channel, + taskExecuteRunningMessage.getMessageSenderAddress()); taskEventService.addEvent(taskEvent); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java index 2f395fd5b0..c1520b07df 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteStartCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteThreadPool; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -34,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -53,13 +51,18 @@ public class TaskExecuteStartProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_EXECUTE_START == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteStartCommand taskExecuteStartCommand = JSONUtils.parseObject(command.getBody(), TaskExecuteStartCommand.class); + Preconditions.checkArgument(CommandType.TASK_EXECUTE_START == command.getType(), + String.format("invalid command type : %s", command.getType())); + TaskExecuteStartCommand taskExecuteStartCommand = + JSONUtils.parseObject(command.getBody(), TaskExecuteStartCommand.class); logger.info("taskExecuteStartCommand: {}", taskExecuteStartCommand); - TaskDefinition taskDefinition = processService.findTaskDefinition(taskExecuteStartCommand.getTaskDefinitionCode(), taskExecuteStartCommand.getTaskDefinitionVersion()); + TaskDefinition taskDefinition = processService.findTaskDefinition( + taskExecuteStartCommand.getTaskDefinitionCode(), taskExecuteStartCommand.getTaskDefinitionVersion()); if (taskDefinition == null) { - logger.error("Task definition can not be found, taskDefinitionCode:{}, taskDefinitionVersion:{}", taskExecuteStartCommand.getTaskDefinitionCode(), taskExecuteStartCommand.getTaskDefinitionVersion()); + logger.error("Task definition can not be found, taskDefinitionCode:{}, taskDefinitionVersion:{}", + taskExecuteStartCommand.getTaskDefinitionCode(), + taskExecuteStartCommand.getTaskDefinitionVersion()); return; } streamTaskExecuteThreadPool.execute(new StreamTaskExecuteRunnable(taskDefinition, taskExecuteStartCommand)); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java index 6079329b90..1ff8cd2966 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java @@ -28,7 +28,6 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -48,11 +47,13 @@ public class TaskKillResponseProcessor implements NettyRequestProcessor { */ @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); + Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), + String.format("invalid command type : %s", command.getType())); - TaskKillResponseCommand responseCommand = JSONUtils.parseObject(command.getBody(), TaskKillResponseCommand.class); + TaskKillResponseCommand responseCommand = + JSONUtils.parseObject(command.getBody(), TaskKillResponseCommand.class); logger.info("[TaskInstance-{}] Received task kill response command : {}", - responseCommand.getTaskInstanceId(), responseCommand); + responseCommand.getTaskInstanceId(), responseCommand); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java index 5ed1a9262a..b023d4ded6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java @@ -25,13 +25,13 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -53,11 +53,13 @@ public class TaskRecallProcessor implements NettyRequestProcessor { */ @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_REJECT == command.getType(), String.format("invalid command type : %s", command.getType())); + Preconditions.checkArgument(CommandType.TASK_REJECT == command.getType(), + String.format("invalid command type : %s", command.getType())); TaskRejectCommand recallCommand = JSONUtils.parseObject(command.getBody(), TaskRejectCommand.class); TaskEvent taskEvent = TaskEvent.newRecallEvent(recallCommand, channel); try { - LoggerUtils.setWorkflowAndTaskInstanceIDMDC(recallCommand.getProcessInstanceId(), recallCommand.getTaskInstanceId()); + LoggerUtils.setWorkflowAndTaskInstanceIDMDC(recallCommand.getProcessInstanceId(), + recallCommand.getTaskInstanceId()); logger.info("Receive task recall command: {}", recallCommand); taskEventService.addEvent(taskEvent); } finally { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java index c8f70d96d0..1408b8d42f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java @@ -34,7 +34,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -50,13 +49,16 @@ public class WorkflowExecutingDataRequestProcessor implements NettyRequestProces @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.WORKFLOW_EXECUTING_DATA_REQUEST == command.getType(), String.format("invalid command type: %s", command.getType())); + Preconditions.checkArgument(CommandType.WORKFLOW_EXECUTING_DATA_REQUEST == command.getType(), + String.format("invalid command type: %s", command.getType())); - WorkflowExecutingDataRequestCommand requestCommand = JSONUtils.parseObject(command.getBody(), WorkflowExecutingDataRequestCommand.class); + WorkflowExecutingDataRequestCommand requestCommand = + JSONUtils.parseObject(command.getBody(), WorkflowExecutingDataRequestCommand.class); logger.info("received command, processInstanceId:{}", requestCommand.getProcessInstanceId()); - Optional workflowExecuteDtoOptional = executingService.queryWorkflowExecutingData(requestCommand.getProcessInstanceId()); + Optional workflowExecuteDtoOptional = + executingService.queryWorkflowExecutingData(requestCommand.getProcessInstanceId()); WorkflowExecutingDataResponseCommand responseCommand = new WorkflowExecutingDataResponseCommand(); workflowExecuteDtoOptional.ifPresent(responseCommand::setWorkflowExecuteDto); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java index 75cf8a9d0b..bed95868ce 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.processor.queue; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand; @@ -26,18 +25,22 @@ import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.service.utils.LoggerUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import io.netty.channel.Channel; + @Component public class StateEventResponseService { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java index 878b8bc6d3..f7b9b033f5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java @@ -20,19 +20,21 @@ package org.apache.dolphinscheduler.server.master.processor.queue; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + /** * task manager */ @@ -152,4 +154,4 @@ public class TaskEventService { } } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java index cad01a325b..0a33410f2e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.server.master.event.TaskEventHandleError; import org.apache.dolphinscheduler.server.master.event.TaskEventHandleException; import org.apache.dolphinscheduler.server.master.event.TaskEventHandler; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; @@ -60,14 +61,14 @@ public class TaskExecuteRunnable implements Runnable { } catch (TaskEventHandleException taskEventHandleException) { // we don't need to resubmit this event, since the worker will resubmit this event logger.error("Handle task event failed, this event will be retry later, event: {}", event, - taskEventHandleException); + taskEventHandleException); } catch (TaskEventHandleError taskEventHandleError) { logger.error("Handle task event error, this event will be removed, event: {}", event, - taskEventHandleError); + taskEventHandleError); events.remove(event); } catch (Exception unknownException) { logger.error("Handle task event error, get a unknown exception, this event will be removed, event: {}", - event, unknownException); + event, unknownException); events.remove(event); } finally { LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); @@ -93,7 +94,8 @@ public class TaskExecuteRunnable implements Runnable { public boolean addEvent(TaskEvent event) { if (event.getProcessInstanceId() != this.processInstanceId) { - logger.warn("event would be abounded, task instance id:{}, process instance id:{}, this.processInstanceId:{}", + logger.warn( + "event would be abounded, task instance id:{}, process instance id:{}, this.processInstanceId:{}", event.getTaskInstanceId(), event.getProcessInstanceId(), this.processInstanceId); return false; } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java index 80d2f12d04..ece37e7e40 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java @@ -71,13 +71,15 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor { this.setMaxPoolSize(masterConfig.getExecThreads()); this.setCorePoolSize(masterConfig.getExecThreads()); taskEventHandlerList.forEach( - taskEventHandler -> taskEventHandlerMap.put(taskEventHandler.getHandleEventType(), taskEventHandler)); + taskEventHandler -> taskEventHandlerMap.put(taskEventHandler.getHandleEventType(), taskEventHandler)); } public void submitTaskEvent(TaskEvent taskEvent) { // stream task event handle - if (taskEvent.getProcessInstanceId() == 0 && streamTaskInstanceExecCacheManager.contains(taskEvent.getTaskInstanceId())) { - streamTaskInstanceExecCacheManager.getByTaskInstanceId(taskEvent.getTaskInstanceId()).addTaskEvent(taskEvent); + if (taskEvent.getProcessInstanceId() == 0 + && streamTaskInstanceExecCacheManager.contains(taskEvent.getTaskInstanceId())) { + streamTaskInstanceExecCacheManager.getByTaskInstanceId(taskEvent.getTaskInstanceId()) + .addTaskEvent(taskEvent); return; } if (!processInstanceExecCacheManager.contains(taskEvent.getProcessInstanceId())) { @@ -85,7 +87,7 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor { return; } TaskExecuteRunnable taskExecuteRunnable = taskExecuteThreadMap.computeIfAbsent(taskEvent.getProcessInstanceId(), - (processInstanceId) -> new TaskExecuteRunnable(processInstanceId, taskEventHandlerMap)); + (processInstanceId) -> new TaskExecuteRunnable(processInstanceId, taskEventHandlerMap)); taskExecuteRunnable.addEvent(taskEvent); } @@ -105,14 +107,16 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor { multiThreadFilterMap.put(taskExecuteThread.getKey(), taskExecuteThread); ListenableFuture future = this.submitListenable(taskExecuteThread::run); future.addCallback(new ListenableFutureCallback() { + @Override public void onFailure(Throwable ex) { Integer processInstanceId = taskExecuteThread.getProcessInstanceId(); logger.error("[WorkflowInstance-{}] persist event failed", processInstanceId, ex); if (!processInstanceExecCacheManager.contains(processInstanceId)) { taskExecuteThreadMap.remove(processInstanceId); - logger.info("[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", - processInstanceId); + logger.info( + "[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", + processInstanceId); } multiThreadFilterMap.remove(taskExecuteThread.getKey()); } @@ -123,8 +127,9 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor { logger.info("[WorkflowInstance-{}] persist events succeeded", processInstanceId); if (!processInstanceExecCacheManager.contains(processInstanceId)) { taskExecuteThreadMap.remove(processInstanceId); - logger.info("[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", - processInstanceId); + logger.info( + "[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", + processInstanceId); } multiThreadFilterMap.remove(taskExecuteThread.getKey()); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java index 1885d82430..545112a699 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.server.master.registry; -import lombok.NonNull; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java index ac8d60e20e..fd835c1234 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java @@ -17,7 +17,9 @@ package org.apache.dolphinscheduler.server.master.registry; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_NODE; +import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; + import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; @@ -28,14 +30,14 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import org.apache.commons.lang3.StringUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_NODE; -import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; - /** *

DolphinScheduler master register client, used to connect to registry and hand the registry events. *

When the Master node startup, it will register in registry center. And start a {@link MasterHeartBeatTask} to update its metadata in registry. diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java index 1eafb4cb54..c6f641bcfd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java @@ -47,11 +47,11 @@ public class MasterRegistryDataListener implements SubscribeListener { if (Strings.isNullOrEmpty(path)) { return; } - //monitor master + // monitor master if (path.startsWith(REGISTRY_DOLPHINSCHEDULER_MASTERS + Constants.SINGLE_SLASH)) { handleMasterEvent(event); } else if (path.startsWith(REGISTRY_DOLPHINSCHEDULER_WORKERS + Constants.SINGLE_SLASH)) { - //monitor worker + // monitor worker handleWorkerEvent(event); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java index 1b1f2c84a8..da2aff1377 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.registry; import org.apache.dolphinscheduler.registry.api.StrategyType; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java index 654d96f25b..4f88ce4787 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java @@ -29,14 +29,15 @@ import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer; import org.apache.dolphinscheduler.server.master.runner.StateWheelExecuteThread; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import java.time.Duration; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import java.time.Duration; - /** * This strategy will change the server status to {@link ServerStatus#WAITING} when disconnect from {@link Registry}. */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java index e1c87de8f3..0d8d4d2815 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import java.util.concurrent.TimeUnit; import org.slf4j.Logger; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java index 5546b474d7..7a488367b0 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.service.MasterFailoverService; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java index 89bc42e61f..377dff7191 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java @@ -41,6 +41,7 @@ import org.apache.dolphinscheduler.service.alert.ProcessAlertManager; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -135,7 +136,8 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCl try { if (!ServerLifeCycleManager.isRunning()) { // the current server is not at running status, cannot consume command. - logger.warn("The current server {} is not at running status, cannot consumes commands.", this.masterAddress); + logger.warn("The current server {} is not at running status, cannot consumes commands.", + this.masterAddress); Thread.sleep(Constants.SLEEP_TIME_MILLIS); } // todo: if the workflow event queue is much, we need to handle the back pressure diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java index 28d31f060a..1a428ee216 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.master.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; -import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -35,16 +34,20 @@ import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +import javax.annotation.PostConstruct; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import java.util.Optional; -import java.util.concurrent.ConcurrentLinkedQueue; - /** * Check thread * 1. timeout task check diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java index 23d4ca8521..29977317c4 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java @@ -17,10 +17,9 @@ package org.apache.dolphinscheduler.server.master.runner; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import javax.annotation.PostConstruct; import org.slf4j.Logger; @@ -60,6 +59,7 @@ public class StreamTaskExecuteThreadPool extends ThreadPoolTaskExecutor { int taskInstanceId = streamTaskExecuteRunnable.getTaskInstance().getId(); ListenableFuture future = this.submitListenable(streamTaskExecuteRunnable::handleEvents); future.addCallback(new ListenableFutureCallback() { + @Override public void onFailure(Throwable ex) { LoggerUtils.setTaskInstanceIdMDC(taskInstanceId); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java index 1842885348..9a422970b4 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java @@ -28,16 +28,18 @@ import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandler; import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; import org.apache.dolphinscheduler.server.master.event.WorkflowEventType; import org.apache.dolphinscheduler.service.utils.LoggerUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + @Component public class WorkflowEventLooper extends BaseDaemonThread { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index 403135f3bd..1a896bf9c5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -85,6 +85,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; import org.apache.dolphinscheduler.service.utils.DagHelper; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -816,7 +817,8 @@ public class WorkflowExecuteRunnable implements Callable { task.getTaskCode(), task.getState()); if (validTaskMap.containsKey(task.getTaskCode())) { - logger.warn("Have same taskCode taskInstance when init task queue, need to check taskExecutionStatus, taskCode:{}", + logger.warn( + "Have same taskCode taskInstance when init task queue, need to check taskExecutionStatus, taskCode:{}", task.getTaskCode()); int oldTaskInstanceId = validTaskMap.get(task.getTaskCode()); TaskInstance oldTaskInstance = taskInstanceMap.get(oldTaskInstanceId); @@ -965,7 +967,8 @@ public class WorkflowExecuteRunnable implements Callable { taskInstance.getProcessInstanceId(), taskInstance.getTaskGroupPriority()); if (!acquireTaskGroup) { - logger.info("Submitted task will not be dispatch right now because the first time to try to acquire" + + logger.info( + "Submitted task will not be dispatch right now because the first time to try to acquire" + " task group failed, taskInstanceName: {}, taskGroupId: {}", taskInstance.getName(), taskGroupId); return Optional.of(taskInstance); @@ -974,7 +977,8 @@ public class WorkflowExecuteRunnable implements Callable { boolean dispatchSuccess = taskProcessor.action(TaskAction.DISPATCH); if (!dispatchSuccess) { - logger.error("Dispatch standby process {} task {} failed", processInstance.getName(), taskInstance.getName()); + logger.error("Dispatch standby process {} task {} failed", processInstance.getName(), + taskInstance.getName()); return Optional.empty(); } taskProcessor.action(TaskAction.RUN); @@ -1431,9 +1435,10 @@ public class WorkflowExecuteRunnable implements Callable { List nextTaskList = DagHelper.parseConditionTask(dependNodeName, skipTaskNodeMap, dag, getCompleteTaskInstanceMap()); if (!nextTaskList.contains(nextNodeName)) { - logger.info("DependTask is a condition task, and its next condition branch does not hava current task, " + - "dependTaskCode: {}, currentTaskCode: {}", dependNodeName, nextNodeName - ); + logger.info( + "DependTask is a condition task, and its next condition branch does not hava current task, " + + "dependTaskCode: {}, currentTaskCode: {}", + dependNodeName, nextNodeName); return false; } } else { @@ -1804,7 +1809,8 @@ public class WorkflowExecuteRunnable implements Callable { TaskInstance retryTask = processService.findTaskInstanceById(task.getId()); if (retryTask != null && retryTask.getState().isForceSuccess()) { task.setState(retryTask.getState()); - logger.info("Task {} has been forced success, put it into complete task list and stop retrying, taskInstanceId: {}", + logger.info( + "Task {} has been forced success, put it into complete task list and stop retrying, taskInstanceId: {}", task.getName(), task.getId()); removeTaskFromStandbyList(task); completeTaskMap.put(task.getTaskCode(), task.getId()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java index 388637a567..20fe003fc3 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.runner; -import com.google.common.base.Strings; -import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.utils.NetUtils; @@ -34,6 +32,14 @@ import org.apache.dolphinscheduler.server.master.event.StateEvent; import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.PostConstruct; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,9 +48,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.concurrent.ListenableFuture; import org.springframework.util.concurrent.ListenableFutureCallback; -import javax.annotation.PostConstruct; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import com.google.common.base.Strings; /** * Used to execute {@link WorkflowExecuteRunnable}. diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java index fc4c4c337d..170f0ac482 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.runner.task; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; @@ -39,6 +38,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Date; import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.auto.service.AutoService; /** @@ -183,13 +183,15 @@ public class CommonTaskProcessor extends BaseTaskProcessor { } protected void convertExeEnvironmentOnlineToTest() { - //SQL taskType + // SQL taskType if (TaskConstants.TASK_TYPE_SQL.equals(taskInstance.getTaskType())) { - //replace test data source - Map taskDefinitionParams = JSONUtils.parseObject(taskInstance.getTaskDefine().getTaskParams(), new TypeReference>() { - }); - Map taskInstanceParams = JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference>() { - }); + // replace test data source + Map taskDefinitionParams = JSONUtils.parseObject( + taskInstance.getTaskDefine().getTaskParams(), new TypeReference>() { + }); + Map taskInstanceParams = + JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference>() { + }); Integer onlineDataSourceId = (Integer) taskDefinitionParams.get(Constants.DATASOUCE); Integer testDataSourceId = processService.queryTestDataSourceId(onlineDataSourceId); taskDefinitionParams.put(Constants.DATASOUCE, testDataSourceId); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java index d9ca82a45c..8bfbde6848 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java @@ -30,12 +30,15 @@ import lombok.NonNull; @Data @AllArgsConstructor public class TaskInstanceKey { + private final int processInstanceId; private final long taskCode; private final int taskVersion; - public static TaskInstanceKey getTaskInstanceKey(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) { - return new TaskInstanceKey(processInstance.getId(), taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()); + public static TaskInstanceKey getTaskInstanceKey(@NonNull ProcessInstance processInstance, + @NonNull TaskInstance taskInstance) { + return new TaskInstanceKey(processInstance.getId(), taskInstance.getTaskCode(), + taskInstance.getTaskDefinitionVersion()); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java index f585c94bbf..793cfacb0e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java @@ -17,18 +17,21 @@ package org.apache.dolphinscheduler.server.master.runner.task; -import lombok.experimental.UtilityClass; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.common.Constants.COMMON_TASK_TYPE; + import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.StringUtils; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static org.apache.dolphinscheduler.common.Constants.COMMON_TASK_TYPE; +import lombok.experimental.UtilityClass; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * the factory to create task processor @@ -47,10 +50,12 @@ public final class TaskProcessorFactory { for (Map.Entry entry : prioritySPIFactory.getSPIMap().entrySet()) { try { logger.info("Registering task processor: {} - {}", entry.getKey(), entry.getValue().getClass()); - PROCESS_MAP.put(entry.getKey(), (Constructor) entry.getValue().getClass().getConstructor()); + PROCESS_MAP.put(entry.getKey(), + (Constructor) entry.getValue().getClass().getConstructor()); logger.info("Registered task processor: {} - {}", entry.getKey(), entry.getValue().getClass()); } catch (NoSuchMethodException e) { - throw new IllegalArgumentException(String.format("The task processor: %s should has a no args constructor", entry.getKey())); + throw new IllegalArgumentException( + String.format("The task processor: %s should has a no args constructor", entry.getKey())); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java index fa85b66f82..90be9f6d3c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java @@ -49,7 +49,8 @@ public class ExecutingService { private ProcessInstanceExecCacheManager processInstanceExecCacheManager; public Optional queryWorkflowExecutingData(Integer processInstanceId) { - WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); + WorkflowExecuteRunnable workflowExecuteRunnable = + processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); if (workflowExecuteRunnable == null) { logger.info("workflow execute data not found, maybe it has finished, workflow id:{}", processInstanceId); return Optional.empty(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java index 05a54e8529..01d9e1968a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java @@ -19,17 +19,18 @@ package org.apache.dolphinscheduler.server.master.service; import org.apache.dolphinscheduler.common.enums.NodeType; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import lombok.NonNull; - /** * failover service */ @Component public class FailoverService { + private static final Logger LOGGER = LoggerFactory.getLogger(FailoverService.class); private final MasterFailoverService masterFailoverService; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java index a4c89b210a..45bf0d74cc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.task; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.model.BaseHeartBeatTask; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -27,6 +25,9 @@ import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class MasterHeartBeatTask extends BaseHeartBeatTask { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java index 69c2318e08..e318e4094a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java @@ -153,7 +153,8 @@ public class DependentExecute { private DependResult getDependTaskResult(long taskCode, ProcessInstance processInstance, int testFlag) { DependResult result; TaskInstance taskInstance = null; - List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId(), testFlag); + List taskInstanceList = + processService.findValidTaskListByProcessId(processInstance.getId(), testFlag); for (TaskInstance task : taskInstanceList) { if (task.getTaskCode() == taskCode) { @@ -191,7 +192,8 @@ public class DependentExecute { ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval(definitionCode, dateInterval, testFlag); - ProcessInstance lastManualProcess = processService.findLastManualProcessInterval(definitionCode, dateInterval, testFlag); + ProcessInstance lastManualProcess = + processService.findLastManualProcessInterval(definitionCode, dateInterval, testFlag); if (lastManualProcess == null) { return lastSchedulerProcess; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java index d23ebacf4c..f63f99d5c6 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java @@ -22,6 +22,7 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; public class SwitchTaskUtils { + private static ScriptEngineManager manager; private static ScriptEngine engine; @@ -35,4 +36,4 @@ public class SwitchTaskUtils { return (Boolean) result; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java index d6a10f97d3..035a811f76 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -34,6 +34,12 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -42,11 +48,6 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.time.Duration; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class ConditionsTaskTest { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java index e2b14714a6..31d73ed12a 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java @@ -35,6 +35,11 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.time.Duration; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -44,10 +49,6 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.time.Duration; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * DependentTaskTest */ diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 88518281cb..ab4e1f9626 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.server.master; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.util.Calendar; import java.util.Date; @@ -34,20 +34,20 @@ import org.slf4j.LoggerFactory; */ public class ParamsTest { - private static final Logger logger = LoggerFactory.getLogger(ParamsTest.class); + private static final Logger logger = LoggerFactory.getLogger(ParamsTest.class); @Test public void systemParamsTest() throws Exception { String command = "${system.biz.date}"; // start process - Map timeParams = BusinessTimeUtils + Map timeParams = BusinessTimeUtils .getBusinessTime(CommandType.START_PROCESS, new Date(), null); command = ParameterUtils.convertParameterPlaceholders(command, timeParams); - logger.info("start process : {}",command); + logger.info("start process : {}", command); Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); @@ -59,7 +59,7 @@ public class ParamsTest { .getBusinessTime(CommandType.COMPLEMENT_DATA, calendar.getTime(), null); command = ParameterUtils.convertParameterPlaceholders(command, timeParams); - logger.info("complement data : {}",command); + logger.info("complement data : {}", command); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java index 902ad406b7..31e55c0af0 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java @@ -30,6 +30,13 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -38,12 +45,6 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class SwitchTaskTest { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java index 1d33316eb2..55b54fd75a 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java @@ -80,4 +80,4 @@ public class ProcessInstanceExecCacheManagerImplTest { Collection workflowExecuteThreads = processInstanceExecCacheManager.getAll(); Assertions.assertEquals(1, workflowExecuteThreads.size()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java index 066a4686cf..bbd4a321cd 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -37,11 +37,11 @@ import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.Date; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java index f14ef4f8be..20a8d2a9f7 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java @@ -54,7 +54,7 @@ public class ExecutorDispatcherTest { public void testDispatchWithException() throws ExecuteException { ExecutionContext executionContext = ExecutionContextTestUtils.getExecutionContext(10000); Assertions.assertThrows(ExecuteException.class, () -> { - executorDispatcher.dispatch(executionContext); + executorDispatcher.dispatch(executionContext); }); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java index 4eba1c5d6d..9fa680fb24 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java @@ -48,6 +48,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; @ExtendWith(SpringExtension.class) @Disabled public class NettyExecutorManagerTest { + @Autowired private NettyExecutorManager nettyExecutorManager; @Test @@ -55,8 +56,9 @@ public class NettyExecutorManagerTest { final NettyServerConfig serverConfig = new NettyServerConfig(); serverConfig.setListenPort(30000); NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(org.apache.dolphinscheduler.remote.command.CommandType.TASK_DISPATCH_REQUEST, - new TaskDispatchProcessor()); + nettyRemotingServer.registerProcessor( + org.apache.dolphinscheduler.remote.command.CommandType.TASK_DISPATCH_REQUEST, + new TaskDispatchProcessor()); nettyRemotingServer.start(); TaskInstance taskInstance = Mockito.mock(TaskInstance.class); ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class); @@ -64,10 +66,10 @@ public class NettyExecutorManagerTest { processInstance.setCommandType(CommandType.COMPLEMENT_DATA); taskInstance.setProcessInstance(processInstance); TaskExecutionContext context = TaskExecutionContextBuilder.get() - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .buildProcessDefinitionRelatedInfo(processDefinition) - .create(); + .buildTaskInstanceRelatedInfo(taskInstance) + .buildProcessInstanceRelatedInfo(processInstance) + .buildProcessDefinitionRelatedInfo(processDefinition) + .create(); ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance); executionContext.setHost(Host.of(NetUtils.getAddr(serverConfig.getListenPort()))); Boolean execute = nettyExecutorManager.execute(executionContext); @@ -96,9 +98,9 @@ public class NettyExecutorManagerTest { } private Command toCommand(TaskExecutionContext taskExecutionContext) { TaskDispatchCommand requestCommand = new TaskDispatchCommand(taskExecutionContext, - "127.0.0.1:5678", - "127.0.0.1:1234", - System.currentTimeMillis()); + "127.0.0.1:5678", + "127.0.0.1:1234", + System.currentTimeMillis()); return requestCommand.convert2Command(); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index a325034c51..4862d28d49 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.ExecutionContextTestUtils; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; + import org.assertj.core.util.Strings; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -29,9 +30,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import com.google.common.collect.Sets; -import org.mockito.junit.jupiter.MockitoExtension; /** * round robin host manager test diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java index 12586c5fd7..8159603544 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java @@ -28,9 +28,12 @@ public class LowerWeightRoundRobinTest { @Test public void testSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 1, System.currentTimeMillis() - 60 * 8 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, System.currentTimeMillis() - 60 * 5 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, System.currentTimeMillis() - 60 * 2 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 1, + System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, + System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, + System.currentTimeMillis() - 60 * 2 * 1000)); LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); HostWeight result; @@ -54,10 +57,14 @@ public class LowerWeightRoundRobinTest { @Test public void testWarmUpSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 8 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 5 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 3 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.4:33", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 11 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 3 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.4:33", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 11 * 1000)); LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); HostWeight result; @@ -81,9 +88,12 @@ public class LowerWeightRoundRobinTest { result = roundRobin.doSelect(sources); Assertions.assertNull(result); - sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.14, 1, System.currentTimeMillis() - 60 * 8 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, System.currentTimeMillis() - 60 * 5 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, System.currentTimeMillis() - 60 * 2 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.14, 1, + System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, + System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, + System.currentTimeMillis() - 60 * 2 * 1000)); result = roundRobin.doSelect(sources); Assertions.assertEquals("192.158.2.1", result.getHost().getIp()); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java index 6c079c71bf..baad4f51f4 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java @@ -38,14 +38,16 @@ public class RandomSelectorTest { @Test public void testSelect1() { RandomSelector selector = new RandomSelector(); - HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1:11", 100, "default"), new HostWorker("192.168.1.2:22", 80, "default"))); + HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1:11", 100, "default"), + new HostWorker("192.168.1.2:22", 80, "default"))); Assertions.assertNotNull(result); } @Test public void testSelect() { RandomSelector selector = new RandomSelector(); - HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1", 11, 100, "default"), new HostWorker("192.168.1.2:", 22, 20, "default"))); + HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1", 11, 100, "default"), + new HostWorker("192.168.1.2:", 22, 20, "default"))); Assertions.assertNotNull(result); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java index 2341ba9ec6..910998abf7 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java @@ -40,13 +40,13 @@ public class RoundRobinSelectorTest { public void testSelect1() { RoundRobinSelector selector = new RoundRobinSelector(); List hostOneList = Arrays.asList( - new HostWorker("192.168.1.1", 80, 20, "kris"), - new HostWorker("192.168.1.2", 80, 10, "kris")); + new HostWorker("192.168.1.1", 80, 20, "kris"), + new HostWorker("192.168.1.2", 80, 10, "kris")); List hostTwoList = Arrays.asList( - new HostWorker("192.168.1.1", 80, 20, "kris"), - new HostWorker("192.168.1.2", 80, 10, "kris"), - new HostWorker("192.168.1.3", 80, 10, "kris")); + new HostWorker("192.168.1.1", 80, 20, "kris"), + new HostWorker("192.168.1.2", 80, 10, "kris"), + new HostWorker("192.168.1.3", 80, 10, "kris")); HostWorker result; result = selector.select(hostOneList); @@ -98,7 +98,8 @@ public class RoundRobinSelectorTest { RoundRobinSelector selector = new RoundRobinSelector(); HostWorker result; result = selector.select( - Arrays.asList(new HostWorker("192.168.1.1", 11, 20, "kris"), new HostWorker("192.168.1.2", 22, 80, "kris"))); + Arrays.asList(new HostWorker("192.168.1.1", 11, 20, "kris"), + new HostWorker("192.168.1.2", 22, 80, "kris"))); Assertions.assertEquals("192.168.1.2", result.getIp()); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java index 476bab2c20..c0fe33a137 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java @@ -30,9 +30,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import io.netty.channel.Channel; -import org.mockito.junit.jupiter.MockitoExtension; /** * task ack processor test diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java index 79065569e5..df88de34e3 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java @@ -33,4 +33,4 @@ public class TaskResponseProcessorTestConfig { public DataQualityResultOperator dataQualityResultOperator() { return Mockito.mock(DataQualityResultOperator.class); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java index 1825d521f2..4b9e37a601 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java @@ -33,9 +33,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import io.netty.channel.Channel; -import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class TaskResponseServiceTest { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java index 77091111ad..0217748fa4 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java @@ -17,16 +17,29 @@ package org.apache.dolphinscheduler.server.master.runner.task; +import static org.mockito.ArgumentMatchers.any; + import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -37,14 +50,6 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; - - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class CommonTaskProcessorTest { @@ -126,7 +131,8 @@ public class CommonTaskProcessorTest { resource.setFileName("fileName"); resourcesList.add(resource); Mockito.doReturn(resourcesList).when(processService).listResourceByIds(new Integer[]{123}); - Mockito.doReturn("tenantCode").when(processService).queryTenantCodeByResName(resource.getFullName(), ResourceType.FILE); + Mockito.doReturn("tenantCode").when(processService).queryTenantCodeByResName(resource.getFullName(), + ResourceType.FILE); Assertions.assertNotNull(map); } @@ -166,17 +172,19 @@ public class CommonTaskProcessorTest { TaskInstance taskInstance = new TaskInstance(); taskInstance.setTestFlag(1); taskInstance.setTaskType("SQL"); - taskInstance.setTaskParams("{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); + taskInstance.setTaskParams( + "{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); TaskDefinition taskDefinition = new TaskDefinition(); - taskDefinition.setTaskParams("{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); + taskDefinition.setTaskParams( + "{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); taskInstance.setTaskDefine(taskDefinition); commonTaskProcessor1.taskInstance = taskInstance; - //The data source instance has no bound test data source + // The data source instance has no bound test data source Mockito.when(processService.queryTestDataSourceId(any(Integer.class))).thenReturn(null); commonTaskProcessor1.convertExeEnvironmentOnlineToTest(); - //The data source instance has bound test data source + // The data source instance has bound test data source Mockito.when(processService.queryTestDataSourceId(any(Integer.class))).thenReturn(2); commonTaskProcessor1.convertExeEnvironmentOnlineToTest(); } diff --git a/dolphinscheduler-meter/pom.xml b/dolphinscheduler-meter/pom.xml index acaee02dca..9fbb578f8c 100644 --- a/dolphinscheduler-meter/pom.xml +++ b/dolphinscheduler-meter/pom.xml @@ -18,18 +18,29 @@ ~ under the License. ~ --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-meter + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + org.springframework.boot @@ -50,8 +61,8 @@ spring-boot-starter-tomcat - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -83,18 +94,6 @@ - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java index 64259f2f4f..e3b140a578 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java @@ -47,6 +47,7 @@ import io.micrometer.core.instrument.MeterRegistry; @EnableAutoConfiguration @ConditionalOnProperty(prefix = "metrics", name = "enabled", havingValue = "true") public class MeterConfiguration { + @Bean public TimedAspect timedAspect(MeterRegistry registry) { return new TimedAspect(registry); diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml index 9ee35e09d0..f3447b319c 100644 --- a/dolphinscheduler-microbench/pom.xml +++ b/dolphinscheduler-microbench/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-microbench jar @@ -35,6 +34,18 @@ benchmarks + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + @@ -67,18 +78,6 @@ - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - @@ -112,5 +111,4 @@ - diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java index 2059f3f266..392ad3e100 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java @@ -122,4 +122,3 @@ public abstract class AbstractBaseBenchmark { } } - diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java index dcce5368e3..b2299458f5 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java @@ -16,13 +16,22 @@ */ package org.apache.dolphinscheduler.microbench.common; - import org.apache.dolphinscheduler.microbench.base.AbstractBaseBenchmark; -import org.openjdk.jmh.annotations.*; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + /** *Enum values JMH test */ @@ -32,13 +41,12 @@ import java.util.concurrent.TimeUnit; public class EnumBenchMark extends AbstractBaseBenchmark { @Benchmark - public boolean simpleTest(){ + public boolean simpleTest() { return Boolean.TRUE; } @Param({"101", "108", "103", "104", "105", "103"}) private int testNum; - @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) @@ -53,8 +61,7 @@ public class EnumBenchMark extends AbstractBaseBenchmark { TestTypeEnum.newGetNameByType(testNum); } - - public enum TestTypeEnum { + public enum TestTypeEnum { TYPE_101(101, "TYPE101"), TYPE_102(102, "TYPE102"), @@ -72,12 +79,10 @@ public class EnumBenchMark extends AbstractBaseBenchmark { return code; } - public String getName() { return name; } - TestTypeEnum(int code, String name) { this.code = code; this.name = name; @@ -87,7 +92,7 @@ public class EnumBenchMark extends AbstractBaseBenchmark { static { for (TestTypeEnum testTypeEnum : TestTypeEnum.values()) { - TEST_TYPE_MAP.put(testTypeEnum.code,testTypeEnum); + TEST_TYPE_MAP.put(testTypeEnum.code, testTypeEnum); } } diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java index ecc54f8f26..1f3f3213d1 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java @@ -43,6 +43,7 @@ import org.openjdk.jmh.annotations.Warmup; @State(Scope.Benchmark) @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) public class RpcTest extends AbstractBaseBenchmark { + private NettyServer nettyServer; private IUserService userService; diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java index bb32093f91..130bcf7469 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; * UserCallback */ public class UserCallback extends AbstractRpcCallBack { + @Override public void run(Object object) { diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml index 410180e847..a3133a52e7 100644 --- a/dolphinscheduler-python/pom.xml +++ b/dolphinscheduler-python/pom.xml @@ -15,7 +15,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 @@ -25,8 +24,8 @@ dev-SNAPSHOT dolphinscheduler-python - ${project.artifactId} jar + ${project.artifactId} @@ -45,10 +44,10 @@ python-api-prepare - prepare-package exec + prepare-package python ${project.basedir}/pydolphinscheduler @@ -64,10 +63,10 @@ python-api-clean - prepare-package exec + prepare-package python ${project.basedir}/pydolphinscheduler @@ -79,10 +78,10 @@ python-api-build - prepare-package exec + prepare-package python ${project.basedir}/pydolphinscheduler @@ -95,10 +94,10 @@ python-pkg-rename-tar - prepare-package exec + prepare-package bash ${project.basedir}/pydolphinscheduler @@ -110,10 +109,10 @@ python-pkg-rename-whl - prepare-package exec + prepare-package bash ${project.basedir}/pydolphinscheduler @@ -125,10 +124,10 @@ sign-source - prepare-package exec + prepare-package ${python.sign.skip} bash @@ -142,10 +141,10 @@ sign-wheel - prepare-package exec + prepare-package ${python.sign.skip} bash diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md index fbc1e93532..eac4b3678a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md +++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md @@ -249,7 +249,8 @@ When you add a new package in pydolphinscheduler, you should also add the packag When you change public class, method or interface, you should change the [UPDATING.md](./UPDATING.md) to notice users who may use it in other way. - +## Reference + [py4j]: https://www.py4j.org/index.html [pycharm]: https://www.jetbrains.com/pycharm [idea]: https://www.jetbrains.com/idea/ @@ -261,3 +262,4 @@ users who may use it in other way. [coverage]: https://coverage.readthedocs.io/en/stable/ [isort]: https://pycqa.github.io/isort/index.html [sphinx]: https://www.sphinx-doc.org/en/master + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml index 682e668301..bc55198b25 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-registry dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-all @@ -44,4 +43,4 @@ ${project.version} - \ No newline at end of file + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml index b704c41316..e798920ceb 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml @@ -17,16 +17,14 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 - dolphinscheduler-registry org.apache.dolphinscheduler + dolphinscheduler-registry dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-api diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java index e76cd57181..5457a9e44a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.registry.api; -import lombok.Data; - import java.time.Duration; +import lombok.Data; + @Data public class ConnectStrategyProperties { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java index 1a3d051091..d854eaf8e1 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java @@ -25,5 +25,6 @@ package org.apache.dolphinscheduler.registry.api; */ @FunctionalInterface public interface ConnectionListener { + void onUpdate(ConnectionState newState); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java index 32fe933fa2..9d68238197 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.registry.api; public class Event { + // The prefix which is watched private String key; // The full path where the event was generated @@ -80,7 +81,8 @@ public class Event { } public String toString() { - return "Event(key=" + this.key() + ", path=" + this.path() + ", data=" + this.data() + ", type=" + this.type() + ")"; + return "Event(key=" + this.key() + ", path=" + this.path() + ", data=" + this.data() + ", type=" + this.type() + + ")"; } public enum Type { @@ -90,6 +92,7 @@ public class Event { } public static class EventBuilder { + private String key; private String path; private String data; @@ -123,7 +126,8 @@ public class Event { } public String toString() { - return "Event.EventBuilder(key=" + this.key + ", path=" + this.path + ", data=" + this.data + ", type=" + this.type + ")"; + return "Event.EventBuilder(key=" + this.key + ", path=" + this.path + ", data=" + this.data + ", type=" + + this.type + ")"; } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java index 2432eb1e25..1642781234 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java @@ -20,5 +20,6 @@ package org.apache.dolphinscheduler.registry.api; public interface SubscribeListener { + void notify(Event event); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml index f55aa022e3..b084db1ccf 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-registry-plugins org.apache.dolphinscheduler + dolphinscheduler-registry-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-etcd @@ -64,4 +63,4 @@ slf4j-api - \ No newline at end of file + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java index 31c77e46eb..56c60c0e4e 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java @@ -29,10 +29,10 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import io.etcd.jetcd.Client; public class EtcdConnectionStateListener implements AutoCloseable { + private final List connectionListeners = Collections.synchronizedList(new ArrayList<>()); // A thread pool that periodically obtains connection status private final ScheduledExecutorService scheduledExecutorService; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 9348d9e027..55be08784f 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -37,6 +37,8 @@ import java.util.stream.Collectors; import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -59,7 +61,6 @@ import io.etcd.jetcd.options.PutOption; import io.etcd.jetcd.options.WatchOption; import io.etcd.jetcd.support.Observers; import io.etcd.jetcd.watch.WatchEvent; -import lombok.NonNull; /** * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to @@ -68,6 +69,7 @@ import lombok.NonNull; @Component @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") public class EtcdRegistry implements Registry { + private static Logger LOGGER = LoggerFactory.getLogger(EtcdRegistry.class); private final Client client; private EtcdConnectionStateListener etcdConnectionStateListener; @@ -88,7 +90,8 @@ public class EtcdRegistry implements Registry { .retryDelay(registryProperties.getRetryDelay().toMillis()) .retryMaxDelay(registryProperties.getRetryMaxDelay().toMillis()) .retryMaxDuration(registryProperties.getRetryMaxDuration()); - if (StringUtils.hasLength(registryProperties.getUser()) && StringUtils.hasLength(registryProperties.getPassword())) { + if (StringUtils.hasLength(registryProperties.getUser()) + && StringUtils.hasLength(registryProperties.getPassword())) { clientBuilder.user(byteSequence(registryProperties.getUser())); clientBuilder.password(byteSequence(registryProperties.getPassword())); } @@ -129,11 +132,12 @@ public class EtcdRegistry implements Registry { try { ByteSequence watchKey = byteSequence(path); WatchOption watchOption = WatchOption.newBuilder().isPrefix(true).build(); - watcherMap.computeIfAbsent(path, $ -> client.getWatchClient().watch(watchKey, watchOption,watchResponse -> { - for (WatchEvent event : watchResponse.getEvents()) { - listener.notify(new EventAdaptor(event, path)); - } - })); + watcherMap.computeIfAbsent(path, + $ -> client.getWatchClient().watch(watchKey, watchOption, watchResponse -> { + for (WatchEvent event : watchResponse.getEvents()) { + listener.notify(new EventAdaptor(event, path)); + } + })); } catch (Exception e) { throw new RegistryException("Failed to subscribe listener for key: " + path, e); } @@ -190,7 +194,7 @@ public class EtcdRegistry implements Registry { client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { })); PutOption putOption = PutOption.newBuilder().withLeaseId(leaseId).build(); - client.getKVClient().put(byteSequence(key), byteSequence(value),putOption).get(); + client.getKVClient().put(byteSequence(key), byteSequence(value), putOption).get(); } else { client.getKVClient().put(byteSequence(key), byteSequence(value)).get(); } @@ -210,10 +214,10 @@ public class EtcdRegistry implements Registry { try { DeleteOption deleteOption = DeleteOption.newBuilder().isPrefix(true).build(); client.getKVClient().delete(byteSequence(key), deleteOption).get(); - } catch (InterruptedException e) { + } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RegistryException("Failed to delete registry key: " + key, e); - } catch (ExecutionException e) { + } catch (ExecutionException e) { throw new RegistryException("Failed to delete registry key: " + key, e); } } @@ -226,10 +230,12 @@ public class EtcdRegistry implements Registry { // Make sure the string end with '/' // eg:change key = /nodes to /nodes/ String prefix = key.endsWith(FOLDER_SEPARATOR) ? key : key + FOLDER_SEPARATOR; - GetOption getOption = GetOption.newBuilder().isPrefix(true).withSortField(GetOption.SortTarget.KEY).withSortOrder(GetOption.SortOrder.ASCEND).build(); + GetOption getOption = GetOption.newBuilder().isPrefix(true).withSortField(GetOption.SortTarget.KEY) + .withSortOrder(GetOption.SortOrder.ASCEND).build(); try { - List keyValues = client.getKVClient().get(byteSequence(prefix),getOption).get().getKvs(); - return keyValues.stream().map(e -> getSubNodeKeyName(prefix, e.getKey().toString(StandardCharsets.UTF_8))).distinct().collect(Collectors.toList()); + List keyValues = client.getKVClient().get(byteSequence(prefix), getOption).get().getKvs(); + return keyValues.stream().map(e -> getSubNodeKeyName(prefix, e.getKey().toString(StandardCharsets.UTF_8))) + .distinct().collect(Collectors.toList()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RegistryException("etcd get children error", e); @@ -243,14 +249,16 @@ public class EtcdRegistry implements Registry { */ private String getSubNodeKeyName(final String prefix, final String fullPath) { String pathWithoutPrefix = fullPath.substring(prefix.length()); - return pathWithoutPrefix.contains(FOLDER_SEPARATOR) ? pathWithoutPrefix.substring(0, pathWithoutPrefix.indexOf(FOLDER_SEPARATOR)) : pathWithoutPrefix; + return pathWithoutPrefix.contains(FOLDER_SEPARATOR) + ? pathWithoutPrefix.substring(0, pathWithoutPrefix.indexOf(FOLDER_SEPARATOR)) + : pathWithoutPrefix; } @Override public boolean exists(String key) { GetOption getOption = GetOption.newBuilder().withCountOnly(true).build(); try { - if (client.getKVClient().get(byteSequence(key),getOption).get().getCount() >= 1) { + if (client.getKVClient().get(byteSequence(key), getOption).get().getCount() >= 1) { return true; } } catch (InterruptedException e) { @@ -275,13 +283,13 @@ public class EtcdRegistry implements Registry { // keep the lease client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { })); - lockClient.lock(byteSequence(key),leaseId).get(); + lockClient.lock(byteSequence(key), leaseId).get(); // save the leaseId for release Lock if (null == threadLocalLockMap.get()) { threadLocalLockMap.set(new HashMap<>()); } - threadLocalLockMap.get().put(key,leaseId); + threadLocalLockMap.get().put(key, leaseId); return true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -320,6 +328,7 @@ public class EtcdRegistry implements Registry { } static final class EventAdaptor extends Event { + public EventAdaptor(WatchEvent event, String key) { key(key); @@ -341,4 +350,3 @@ public class EtcdRegistry implements Registry { } } } - diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java index 96be83c5d5..593752a7c0 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java @@ -19,17 +19,18 @@ package org.apache.dolphinscheduler.plugin.registry.etcd; import java.time.Duration; +import lombok.Data; + import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import lombok.Data; - @Data @Configuration @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") @ConfigurationProperties(prefix = "registry") public class EtcdRegistryProperties { + private String endpoints; private String namespace = "dolphinscheduler"; private Duration connectionTimeout = Duration.ofSeconds(9); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml index 150b058354..fd7f0925e2 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml @@ -17,15 +17,14 @@ ~ specific language governing permissions and limitations ~ under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-registry-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-mysql @@ -53,4 +52,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java index d475a1ae4f..d5caae9f00 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.registry.mysql; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.registry.mysql.task.EphemeralDateManager; import org.apache.dolphinscheduler.plugin.registry.mysql.task.RegistryLockManager; import org.apache.dolphinscheduler.plugin.registry.mysql.task.SubscribeDataManager; @@ -27,16 +26,19 @@ import org.apache.dolphinscheduler.registry.api.Registry; import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.SubscribeListener; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Collection; + +import javax.annotation.PostConstruct; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import java.sql.SQLException; -import java.time.Duration; -import java.util.Collection; - /** * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on mysql database to * store the DolphinScheduler master/worker's metadata and do the server registry/unRegistry. diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java index d80b8ead0d..9690418bfb 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java @@ -19,14 +19,14 @@ package org.apache.dolphinscheduler.plugin.registry.mysql; import java.time.Duration; +import lombok.Data; + import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import com.zaxxer.hikari.HikariConfig; -import lombok.Data; - @Data @Configuration @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "mysql") diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java index 657c80b716..bacc8c9a8d 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.registry.mysql.model; public enum DataType { + EPHEMERAL(1), PERSISTENT(2), ; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java index 478727545b..9c6ca6eeeb 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java @@ -33,14 +33,14 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; - public class RegistryLockManager implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(RegistryLockManager.class); @@ -112,6 +112,7 @@ public class RegistryLockManager implements AutoCloseable { */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) static class LockTermRefreshTask implements Runnable { + private final Map lockHoldMap; private final MysqlOperator mysqlOperator; @@ -134,4 +135,3 @@ public class RegistryLockManager implements AutoCloseable { } } } - diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java index 4c8d7190f3..bbcc25d7be 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java @@ -33,13 +33,13 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.RequiredArgsConstructor; - /** * Used to refresh if the subscribe path has been changed. */ @@ -58,7 +58,8 @@ public class SubscribeDataManager implements AutoCloseable { this.mysqlOperator = mysqlOperator; this.dataSubscribeCheckThreadPool = Executors.newScheduledThreadPool( 1, - new ThreadFactoryBuilder().setNameFormat("MysqlRegistrySubscribeDataCheckThread").setDaemon(true).build()); + new ThreadFactoryBuilder().setNameFormat("MysqlRegistrySubscribeDataCheckThread").setDaemon(true) + .build()); } public void start() { @@ -146,8 +147,8 @@ public class SubscribeDataManager implements AutoCloseable { Event.Type type) { for (MysqlRegistryData data : dataList) { if (data.getKey().startsWith(subscribeKey)) { - subscribeListeners.forEach(subscribeListener -> - subscribeListener.notify(new Event(data.getKey(), data.getKey(), data.getData(), type))); + subscribeListeners.forEach(subscribeListener -> subscribeListener + .notify(new Event(data.getKey(), data.getKey(), data.getData(), type))); } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java index c39f6a54b0..ca22f68bf2 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java @@ -27,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public final class ZookeeperConnectionStateListener implements ConnectionStateListener { - + private static final Logger logger = LoggerFactory.getLogger(ZookeeperConnectionStateListener.class); private final ConnectionListener listener; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 5aa99b5198..a2741d4d9a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -17,8 +17,14 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper; -import com.google.common.base.Strings; -import lombok.NonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import org.apache.dolphinscheduler.registry.api.ConnectionListener; +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.Registry; +import org.apache.dolphinscheduler.registry.api.RegistryException; +import org.apache.dolphinscheduler.registry.api.SubscribeListener; + import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; @@ -28,19 +34,11 @@ import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.utils.CloseableUtils; -import org.apache.dolphinscheduler.registry.api.ConnectionListener; -import org.apache.dolphinscheduler.registry.api.Event; -import org.apache.dolphinscheduler.registry.api.Registry; -import org.apache.dolphinscheduler.registry.api.RegistryException; -import org.apache.dolphinscheduler.registry.api.SubscribeListener; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Comparator; @@ -49,7 +47,14 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import javax.annotation.PostConstruct; + +import lombok.NonNull; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import com.google.common.base.Strings; @Component @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java index 84026a34f0..42dbc5d256 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java @@ -27,6 +27,7 @@ import org.springframework.context.annotation.Configuration; @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") @ConfigurationProperties(prefix = "registry") public class ZookeeperRegistryProperties { + private ZookeeperProperties zookeeper = new ZookeeperProperties(); public ZookeeperProperties getZookeeper() { @@ -38,6 +39,7 @@ public class ZookeeperRegistryProperties { } public static final class ZookeeperProperties { + private String namespace; private String connectString; private RetryPolicy retryPolicy = new RetryPolicy(); @@ -103,6 +105,7 @@ public class ZookeeperRegistryProperties { } public static final class RetryPolicy { + private Duration baseSleepTime = Duration.ofMillis(60); private int maxRetries; private Duration maxSleep = Duration.ofMillis(300); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml index 8c3c108575..a2bcfa9775 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml @@ -17,17 +17,15 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-registry dev-SNAPSHOT dolphinscheduler-registry-plugins - 4.0.0 pom diff --git a/dolphinscheduler-registry/pom.xml b/dolphinscheduler-registry/pom.xml index e711900550..7a273dc1a8 100644 --- a/dolphinscheduler-registry/pom.xml +++ b/dolphinscheduler-registry/pom.xml @@ -15,35 +15,17 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry pom - - - org.springframework - spring-context - provided - - - org.springframework.boot - spring-boot-autoconfigure - provided - - - javax.annotation - javax.annotation-api - - - dolphinscheduler-registry-api dolphinscheduler-registry-plugins @@ -61,4 +43,21 @@ + + + + org.springframework + spring-context + provided + + + org.springframework.boot + spring-boot-autoconfigure + provided + + + javax.annotation + javax.annotation-api + + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index d47d68fa3d..5ac5918044 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -17,17 +17,6 @@ package org.apache.dolphinscheduler.remote; -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.timeout.IdleStateHandler; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.command.Command; @@ -46,8 +35,6 @@ import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.remote.utils.NettyUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; @@ -60,6 +47,21 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.Epoll; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.timeout.IdleStateHandler; + public class NettyRemotingClient implements AutoCloseable { private final Logger logger = LoggerFactory.getLogger(NettyRemotingClient.class); @@ -87,9 +89,11 @@ public class NettyRemotingClient implements AutoCloseable { public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; if (Epoll.isAvailable()) { - this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); + this.workerGroup = + new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); } else { - this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); + this.workerGroup = + new NioEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); } this.callbackExecutor = new ThreadPoolExecutor( Constants.CPUS, @@ -101,7 +105,8 @@ public class NettyRemotingClient implements AutoCloseable { new CallerThreadExecutePolicy()); this.clientHandler = new NettyClientHandler(this, callbackExecutor); - this.responseFutureExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ResponseFutureExecutor")); + this.responseFutureExecutor = + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ResponseFutureExecutor")); this.start(); } @@ -117,14 +122,18 @@ public class NettyRemotingClient implements AutoCloseable { .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) .handler(new ChannelInitializer() { + @Override public void initChannel(SocketChannel ch) { ch.pipeline() - .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) + .addLast("client-idle-handler", + new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, + TimeUnit.MILLISECONDS)) .addLast(new NettyDecoder(), clientHandler, encoder); } }); - this.responseFutureExecutor.scheduleAtFixedRate(ResponseFuture::scanFutureTable, 5000, 1000, TimeUnit.MILLISECONDS); + this.responseFutureExecutor.scheduleAtFixedRate(ResponseFuture::scanFutureTable, 5000, 1000, + TimeUnit.MILLISECONDS); isStarted.compareAndSet(false, true); } @@ -148,14 +157,14 @@ public class NettyRemotingClient implements AutoCloseable { */ final long opaque = command.getOpaque(); /* - * control concurrency number + * control concurrency number */ boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore); /* - * response future + * response future */ final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, @@ -184,7 +193,8 @@ public class NettyRemotingClient implements AutoCloseable { throw new RemotingException(String.format("send command to host: %s failed", host), ex); } } else { - String message = String.format("try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", + String message = String.format( + "try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits()); throw new RemotingTooMuchRequestException(message); } @@ -198,7 +208,8 @@ public class NettyRemotingClient implements AutoCloseable { * @param timeoutMillis timeoutMillis * @return command */ - public Command sendSync(final Host host, final Command command, final long timeoutMillis) throws InterruptedException, RemotingException { + public Command sendSync(final Host host, final Command command, + final long timeoutMillis) throws InterruptedException, RemotingException { final Channel channel = getChannel(host); if (channel == null) { throw new RemotingException(String.format("connect to : %s fail", host)); @@ -254,7 +265,8 @@ public class NettyRemotingClient implements AutoCloseable { throw remotingException; } catch (Exception e) { logger.error("Send command {} to address {} encounter error.", command, host.getAddress()); - throw new RemotingException(String.format("Send command : %s , to :%s encounter error", command, host.getAddress()), e); + throw new RemotingException( + String.format("Send command : %s , to :%s encounter error", command, host.getAddress()), e); } } @@ -275,7 +287,8 @@ public class NettyRemotingClient implements AutoCloseable { * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { this.clientHandler.registerProcessor(commandType, processor, executor); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 098751ab65..63c1392a06 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -103,8 +103,10 @@ public class NettyRemotingServer { */ public NettyRemotingServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; - ThreadFactory bossThreadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerBossThread_%s").build(); - ThreadFactory workerThreadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerWorkerThread_%s").build(); + ThreadFactory bossThreadFactory = + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerBossThread_%s").build(); + ThreadFactory workerThreadFactory = + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerWorkerThread_%s").build(); if (Epoll.isAvailable()) { this.bossGroup = new EpollEventLoopGroup(1, bossThreadFactory); this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), workerThreadFactory); @@ -146,7 +148,8 @@ public class NettyRemotingServer { if (future.isSuccess()) { logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); } else if (future.cause() != null) { - throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()), future.cause()); + throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()), + future.cause()); } else { throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort())); } @@ -162,7 +165,8 @@ public class NettyRemotingServer { ch.pipeline() .addLast("encoder", new NettyEncoder()) .addLast("decoder", new NettyDecoder()) - .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) + .addLast("server-idle-handle", + new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) .addLast("handler", serverHandler); } @@ -183,7 +187,8 @@ public class NettyRemotingServer { * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { this.serverHandler.registerProcessor(commandType, processor, executor); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java index 84b5c1f90b..4066b6ed10 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java @@ -35,6 +35,7 @@ import io.netty.handler.codec.ReplayingDecoder; * netty decoder */ public class NettyDecoder extends ReplayingDecoder { + private static final Logger logger = LoggerFactory.getLogger(NettyDecoder.class); public NettyDecoder() { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java index 04487ff686..c1d6e0fcd2 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java @@ -58,4 +58,3 @@ public class NettyEncoder extends MessageToByteEncoder { out.writeBytes(headerBytes); } } - diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 527a269411..28c36e56fb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -33,11 +33,11 @@ public class Command implements Serializable { public static final byte MAGIC = (byte) 0xbabe; public static final byte VERSION = 0; - public Command(){ + public Command() { this.opaque = REQUEST_ID.getAndIncrement(); } - public Command(long opaque){ + public Command(long opaque) { this.opaque = opaque; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java index b83c9ec53c..920332f4a5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java @@ -38,7 +38,6 @@ public enum CommandType { GET_LOG_BYTES_RESPONSE, - WORKER_REQUEST, MASTER_RESPONSE, diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java index f265b758b4..b55453b206 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.Data; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.Data; + /** * process host update */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java index b98e5dad0b..6b21e1acd6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.AllArgsConstructor; -import lombok.Data; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; + /** * db task final result response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java index f04c6107c1..bf9abe8c96 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java @@ -25,7 +25,6 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.ToString; - /** * The task dispatch message, means dispatch a task to worker. */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java index bf0ff595d6..adfcaa105a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * db task final result response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java index fb24438426..42d99672e0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import java.util.Date; - import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -74,13 +72,11 @@ public class TaskExecuteResultCommand extends BaseCommand { */ private String executePath; - /** * end time */ private long endTime; - /** * processId */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java index ea0da08cff..120867977b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.remote.command; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.io.Serializable; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.common.utils.JSONUtils; - -import java.io.Serializable; /** * task execute running ack command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java index ed0231b374..c02ea64849 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.ToString; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; /** * Task running message, means the task is running in worker. diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java index ba7fad8368..7f92b41543 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java @@ -17,16 +17,17 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import java.io.Serializable; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * kill task response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java index c126a45055..55eb272eb3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * task savepoint response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java index 5db49f8500..def9f663ed 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * db task final result response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java index 2acf46d253..5bc4154d9c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.alert; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @NoArgsConstructor @AllArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java index 9c5e62781b..6d729e7539 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.remote.command.alert; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; @@ -27,6 +24,10 @@ import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @NoArgsConstructor @AllArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java index c0867ee7ae..7b2fd98edd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.remote.command.alert; +import java.io.Serializable; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; - @Data @NoArgsConstructor @AllArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java index 26412b8283..91706feb83 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @AllArgsConstructor @NoArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java index 2e54a9008d..0952cfb1ba 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -27,6 +24,10 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @AllArgsConstructor @NoArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 0022706e99..de73e26861 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * get log bytes request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java index 30072e3151..3ba4cf3b58 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * get log bytes response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java index 93efc25318..8dc8f274d8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * remove task log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java index 431a52066d..6e0ae320a6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * remove task log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 4d11836f45..e4075f6cf7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * roll view log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java index 4fea16d912..71be7a0cf3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * roll view log response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index 6eae191128..a4b3c4749b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * view log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java index 52a1b17410..e9a368ad7c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * view log response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java index 18ee88f696..ff41e9b101 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.remote.config; +import org.apache.dolphinscheduler.remote.utils.Constants; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.remote.utils.Constants; @Data @Builder diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java index 1779da3270..0bffe9272b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java @@ -19,15 +19,14 @@ package org.apache.dolphinscheduler.remote.dto; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskExecuteType; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import java.util.Date; import java.util.Map; import lombok.Data; -import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; - @Data public class TaskInstanceExecuteDto { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java index 3eac76da7a..a2b5d54f61 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java @@ -17,7 +17,13 @@ package org.apache.dolphinscheduler.remote.dto; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import java.util.Collection; import java.util.Date; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java index d774dc8b9e..1d025cc268 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java @@ -55,5 +55,4 @@ public class RemotingException extends Exception { super(cause); } - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java index 3d91ba57f6..5788221d6c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java @@ -17,17 +17,15 @@ package org.apache.dolphinscheduler.remote.exceptions; - /** * timeout exception */ -public class RemotingTimeoutException extends RemotingException{ +public class RemotingTimeoutException extends RemotingException { public RemotingTimeoutException(String message) { super(message); } - public RemotingTimeoutException(String address, long timeoutMillis) { this(address, timeoutMillis, null); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java index 82cc3f4dbf..4e31497311 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.remote.exceptions; /** * too much request exception */ -public class RemotingTooMuchRequestException extends RemotingException{ +public class RemotingTooMuchRequestException extends RemotingException { public RemotingTooMuchRequestException(String message) { super(message); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java index 95a04b1f1a..c8284890f9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java @@ -28,13 +28,13 @@ public class ReleaseSemaphore { private final AtomicBoolean released; - public ReleaseSemaphore(Semaphore semaphore){ + public ReleaseSemaphore(Semaphore semaphore) { this.semaphore = semaphore; this.released = new AtomicBoolean(false); } - public void release(){ - if(this.released.compareAndSet(false, true)){ + public void release() { + if (this.released.compareAndSet(false, true)) { this.semaphore.release(); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java index 5e5462c7e8..4e4ae1456c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java @@ -72,7 +72,8 @@ public class ResponseFuture { private Throwable cause; - public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, ReleaseSemaphore releaseSemaphore) { + public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, + ReleaseSemaphore releaseSemaphore) { this.opaque = opaque; this.timeoutMillis = timeoutMillis; this.invokeCallback = invokeCallback; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index ad9787d705..56c9fc70e4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -117,7 +117,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { ExecutorService executorRef = executor; if (executorRef == null) { executorRef = defaultExecutor; @@ -159,7 +160,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { try { pair.getRight().submit(run); } catch (RejectedExecutionException e) { - logger.warn("thread pool is full, discard command {} from {}", command, ChannelUtils.getRemoteAddress(channel)); + logger.warn("thread pool is full, discard command {} from {}", command, + ChannelUtils.getRemoteAddress(channel)); } } else { logger.warn("receive response {}, but not matched any request ", command); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index 2be00c914a..8ae04a70b6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -38,7 +38,6 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.timeout.IdleStateEvent; - /** * netty server request handler */ @@ -55,7 +54,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { /** * server processors queue */ - private final ConcurrentHashMap> processors = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> processors = + new ConcurrentHashMap<>(); public NettyServerHandler(NettyRemotingServer nettyRemotingServer) { this.nettyRemotingServer = nettyRemotingServer; @@ -100,7 +100,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { ExecutorService executorRef = executor; if (executorRef == null) { executorRef = nettyRemotingServer.getDefaultExecutor(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java index 0d623e398a..4609c6d239 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java @@ -44,7 +44,6 @@ public class NettyRemoteChannel { */ private final Host host; - public NettyRemoteChannel(Channel channel, long opaque) { this.channel = channel; this.host = ChannelUtils.toAddress(channel); @@ -69,7 +68,7 @@ public class NettyRemoteChannel { return host; } - public boolean isActive(){ + public boolean isActive() { return this.channel.isActive(); } @@ -77,7 +76,7 @@ public class NettyRemoteChannel { return this.channel.writeAndFlush(command); } - public void close(){ + public void close() { this.channel.close(); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java index 6966b53d17..d3e37f02ee 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java @@ -16,9 +16,10 @@ */ package org.apache.dolphinscheduler.remote.processor; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.remote.command.Command; +import io.netty.channel.Channel; + /** * netty request processor */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java index 32a0f4673d..e35fb44d41 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java @@ -141,8 +141,7 @@ public class StateEventCallbackService { Thread.currentThread().interrupt(); } catch (RemotingException e) { logger.error("send sync fail, host:{}, command:{}", host, requestCommand, e); - } - finally { + } finally { this.nettyRemotingClient.closeChannel(host); } return null; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java index 048ea86acb..318e20d967 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.remote.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * caller thread execute */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 6748db0356..8611dda54c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -49,7 +49,6 @@ public class Constants { */ public static final int CPUS = Runtime.getRuntime().availableProcessors(); - public static final String LOCAL_ADDRESS = NetUtils.getHost(); /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java index e183289f73..e860a72b3b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; * json serialize or deserialize */ public class JsonSerializer { + private static final ObjectMapper objectMapper = new ObjectMapper(); private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java index be84f0f221..69f69c1ba0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java @@ -36,11 +36,11 @@ public class NamedThreadFactory implements ThreadFactory { */ private final int count; - public NamedThreadFactory(String name){ + public NamedThreadFactory(String name) { this(name, 0); } - public NamedThreadFactory(String name, int count){ + public NamedThreadFactory(String name, int count) { this.name = name; this.count = count; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java index 655aa2730e..9857c9f4a3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java @@ -42,7 +42,7 @@ public @interface Rpc { boolean callBack() default false; - //todo It is better to set the timeout period for synchronous calls + // todo It is better to set the timeout period for synchronous calls /** * When it is asynchronous transmission, callback must be set @@ -51,7 +51,4 @@ public @interface Rpc { Class ackCallback() default AbstractRpcCallBack.class; - - - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java index 5ebb2cfe7a..81b16280e0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java @@ -25,5 +25,6 @@ import java.lang.annotation.Target; @Target({ElementType.TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface RpcService { + String value() default ""; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index c001e5a753..d35edcdaea 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -31,13 +31,13 @@ import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; import java.lang.reflect.Method; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import net.bytebuddy.implementation.bind.annotation.AllArguments; import net.bytebuddy.implementation.bind.annotation.Origin; import net.bytebuddy.implementation.bind.annotation.RuntimeType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * ConsumerInterceptor */ @@ -70,7 +70,7 @@ public class ConsumerInterceptor { while (retries-- > 0) { RpcResponse rsp; rsp = nettyClient.sendMsg(host, protocol, async); - //success + // success if (null != rsp && rsp.getStatus() == 0) { return rsp.getResult(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 813bf719f2..5263615994 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -42,7 +42,8 @@ public class NettyDecoder extends ByteToMessageDecoder { } @Override - protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, + List list) throws Exception { if (byteBuf.readableBytes() < RpcProtocolConstants.HEADER_LENGTH) { return; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index 58328134b9..d076e4760a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -32,7 +32,8 @@ import io.netty.handler.codec.MessageToByteEncoder; public class NettyEncoder extends MessageToByteEncoder> { @Override - protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, ByteBuf byteBuf) throws Exception { + protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, + ByteBuf byteBuf) throws Exception { MessageHeader msgHeader = msg.getMsgHeader(); byteBuf.writeShort(msgHeader.getMagic()); byteBuf.writeByte(msgHeader.getEventType()); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java index 558c88067b..5810993fc0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java @@ -19,9 +19,8 @@ package org.apache.dolphinscheduler.rpc.common; public enum RequestEventType { - HEARTBEAT((byte)1,"heartbeat"), - BUSINESS((byte)2,"business request"); - + HEARTBEAT((byte) 1, "heartbeat"), + BUSINESS((byte) 2, "business request"); private Byte type; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java index 66ca157e23..26f44814ed 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.rpc.common;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ public enum ResponseEventType { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java index 1ddd574e0d..1a2ce2efee 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java @@ -33,7 +33,8 @@ public enum ThreadPoolManager { private static final long KEEP_ALIVE_TIME = 60; ThreadPoolManager() { - executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, Runtime.getRuntime().availableProcessors() * 4, KEEP_ALIVE_TIME, TimeUnit.SECONDS, + executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, + Runtime.getRuntime().availableProcessors() * 4, KEEP_ALIVE_TIME, TimeUnit.SECONDS, new ArrayBlockingQueue<>(WORK_QUEUE_SIZE), new DiscardPolicy()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java index e6a85d3b8d..224ab7b869 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java @@ -19,9 +19,9 @@ package org.apache.dolphinscheduler.rpc.protocol; public enum EventType { - HEARTBEAT((byte)1,"heartbeat"), - REQUEST((byte)2,"business request"), - RESPONSE((byte)3,"business response"); + HEARTBEAT((byte) 1, "heartbeat"), + REQUEST((byte) 2, "business request"), + RESPONSE((byte) 3, "business response"); private Byte type; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 1c8e81a79e..633efa21d6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -80,7 +80,6 @@ public class NettyClient { */ private final NettyClientConfig clientConfig; - /** * client bootstrap */ @@ -143,6 +142,7 @@ public class NettyClient { this.clientConfig = clientConfig; if (Epoll.isAvailable()) { this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -152,6 +152,7 @@ public class NettyClient { }); } else { this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -179,12 +180,15 @@ public class NettyClient { .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) .handler(new LoggingHandler(LogLevel.DEBUG)) .handler(new ChannelInitializer() { + @Override public void initChannel(SocketChannel ch) { ch.pipeline() .addLast(new NettyEncoder()) .addLast(new NettyDecoder(RpcResponse.class)) - .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) + .addLast("client-idle-handler", + new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, + TimeUnit.MILLISECONDS)) .addLast(new NettyClientHandler()); } }); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index e810af2ce7..c8baec2074 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -44,7 +44,6 @@ import io.netty.handler.timeout.IdleStateEvent; @ChannelHandler.Sharable public class NettyClientHandler extends ChannelInboundHandlerAdapter { - private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); private static final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; @@ -87,7 +86,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { try { consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); - } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { logger.error("rpc service call back error,serviceName {},rsp {}", serviceName, rsp); } } else { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java index 4652d0a5bb..559eb90e46 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java @@ -86,6 +86,7 @@ public class NettyServer { this.serverConfig = serverConfig; if (Epoll.isAvailable()) { this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -95,15 +96,18 @@ public class NettyServer { }); this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + return new Thread(r, + String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); } }); } else { this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -113,11 +117,13 @@ public class NettyServer { }); this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + return new Thread(r, + String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); } }); } @@ -152,14 +158,18 @@ public class NettyServer { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); - throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + throw new RuntimeException( + String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); } else if (future.cause() != null) { - throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), future.cause()); + throw new RuntimeException( + String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), + future.cause()); } else { - throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + throw new RuntimeException( + String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } } } @@ -173,7 +183,8 @@ public class NettyServer { ch.pipeline() .addLast(new NettyDecoder(RpcRequest.class)) .addLast(new NettyEncoder()) - .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) + .addLast("server-idle-handle", + new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) .addLast("handler", new NettyServerHandler()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java index a5b9250fc1..5ae26b2e03 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.rpc.serializer;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java index 5c29d774a9..3ef5b4ddda 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java @@ -1,25 +1,24 @@ package org.apache.dolphinscheduler.rpc.serializer;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import java.util.HashMap; public enum RpcSerializer { - PROTOSTUFF((byte) 1, new ProtoStuffSerializer()); byte type; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java index 6652a60d7c..ac8f9ac9aa 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.remote; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; @@ -28,13 +25,18 @@ import org.apache.dolphinscheduler.remote.future.InvokeCallback; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Host; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.io.Serializable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; + /** * netty remote client test */ @@ -49,13 +51,13 @@ public class NettyRemotingClientTest { NettyRemotingServer server = new NettyRemotingServer(serverConfig); server.registerProcessor(CommandType.PING, new NettyRequestProcessor() { + @Override public void process(Channel channel, Command command) { channel.writeAndFlush(Pong.create(command.getOpaque())); } }); - server.start(); // final NettyClientConfig clientConfig = new NettyClientConfig(); @@ -75,11 +77,12 @@ public class NettyRemotingClientTest { * test sned async */ @Test - public void testSendAsync(){ + public void testSendAsync() { NettyServerConfig serverConfig = new NettyServerConfig(); NettyRemotingServer server = new NettyRemotingServer(serverConfig); server.registerProcessor(CommandType.PING, new NettyRequestProcessor() { + @Override public void process(Channel channel, Command command) { channel.writeAndFlush(Pong.create(command.getOpaque())); @@ -93,13 +96,15 @@ public class NettyRemotingClientTest { Command commandPing = Ping.create(); try { final AtomicLong opaque = new AtomicLong(0); - client.sendAsync(new Host("127.0.0.1", serverConfig.getListenPort()), commandPing, 2000, new InvokeCallback() { - @Override - public void operationComplete(ResponseFuture responseFuture) { - opaque.set(responseFuture.getOpaque()); - latch.countDown(); - } - }); + client.sendAsync(new Host("127.0.0.1", serverConfig.getListenPort()), commandPing, 2000, + new InvokeCallback() { + + @Override + public void operationComplete(ResponseFuture responseFuture) { + opaque.set(responseFuture.getOpaque()); + latch.countDown(); + } + }); latch.await(); Assertions.assertEquals(commandPing.getOpaque(), opaque.get()); } catch (Exception e) { diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java index cfd5b0673d..e0c3e20e90 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.remote.command.alert; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,7 +31,8 @@ public class AlertSendRequestCommandTest { int groupId = 1; String title = "test-title"; String content = "test-content"; - AlertSendRequestCommand requestCommand = new AlertSendRequestCommand(groupId,title,content,WarningType.FAILURE.getCode()); + AlertSendRequestCommand requestCommand = + new AlertSendRequestCommand(groupId, title, content, WarningType.FAILURE.getCode()); Command command = requestCommand.convert2Command(); Assertions.assertEquals(CommandType.ALERT_SEND_REQUEST, command.getType()); AlertSendRequestCommand verifyCommand = new AlertSendRequestCommand(); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java index 966386d12e..aceefb7980 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java @@ -19,12 +19,13 @@ package org.apache.dolphinscheduler.remote.command.alert; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class AlertSendResponseCommandTest { @Test diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java index 8c47f267ee..bcc6108422 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.enums.CacheType; import org.apache.dolphinscheduler.remote.command.CacheExpireCommand; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java index 1958786120..3cb6fb319c 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java @@ -20,20 +20,23 @@ package org.apache.dolphinscheduler.remote.command.future; import org.apache.dolphinscheduler.remote.future.InvokeCallback; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class ResponseFutureTest { @Test public void testScanFutureTable() { - ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service")); + ScheduledExecutorService executorService = + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service")); executorService.scheduleAtFixedRate(new Runnable() { + @Override public void run() { ResponseFuture.scanFutureTable(); @@ -42,6 +45,7 @@ public class ResponseFutureTest { CountDownLatch latch = new CountDownLatch(1); InvokeCallback invokeCallback = new InvokeCallback() { + @Override public void operationComplete(ResponseFuture responseFuture) { latch.countDown(); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java index 898092c0b8..dd9027f68d 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java index 1a4a6b83a4..a6968174fb 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java index f5380c01c7..802589ccdf 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java index 58a3706a0b..574b795d05 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java index 3e37bfb973..cfda036105 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java index 09c60c4e77..3ff50d666a 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; public class RpcTest { + private NettyServer nettyServer; private IUserService userService; diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml index 230c592827..88a6aef949 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-scheduler-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-scheduler-api @@ -34,4 +33,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java index 36f04c22bb..c5d21cde4f 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.dao.entity.Schedule; /** * This is the interface for scheduler, contains methods to operate schedule task. */ -public interface SchedulerApi extends AutoCloseable{ +public interface SchedulerApi extends AutoCloseable { /** * Start the scheduler, if not start, the scheduler will not execute task. diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml index d3afa0e70f..27e5795f5f 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-scheduler-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-scheduler-quartz @@ -75,4 +74,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java index 61eb86c982..f51264d668 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java @@ -66,16 +66,21 @@ public class ProcessScheduleTask extends QuartzJobBean { // query schedule Schedule schedule = processService.querySchedule(scheduleId); if (schedule == null || ReleaseState.OFFLINE == schedule.getReleaseState()) { - logger.warn("process schedule does not exist in db or process schedule offline，delete schedule job in quartz, projectId:{}, scheduleId:{}", projectId, scheduleId); + logger.warn( + "process schedule does not exist in db or process schedule offline，delete schedule job in quartz, projectId:{}, scheduleId:{}", + projectId, scheduleId); deleteJob(context, projectId, scheduleId); return; } - ProcessDefinition processDefinition = processService.findProcessDefinitionByCode(schedule.getProcessDefinitionCode()); + ProcessDefinition processDefinition = + processService.findProcessDefinitionByCode(schedule.getProcessDefinitionCode()); // release state : online/offline ReleaseState releaseState = processDefinition.getReleaseState(); if (releaseState == ReleaseState.OFFLINE) { - logger.warn("process definition does not exist in db or offline，need not to create command, projectId:{}, processDefinitionId:{}", projectId, processDefinition.getId()); + logger.warn( + "process definition does not exist in db or offline，need not to create command, projectId:{}, processDefinitionId:{}", + projectId, processDefinition.getId()); return; } @@ -87,7 +92,8 @@ public class ProcessScheduleTask extends QuartzJobBean { command.setScheduleTime(scheduledFireTime); command.setStartTime(fireTime); command.setWarningGroupId(schedule.getWarningGroupId()); - String workerGroup = StringUtils.isEmpty(schedule.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP : schedule.getWorkerGroup(); + String workerGroup = StringUtils.isEmpty(schedule.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP + : schedule.getWorkerGroup(); command.setWorkerGroup(workerGroup); command.setWarningType(schedule.getWarningType()); command.setProcessInstancePriority(schedule.getProcessInstancePriority()); diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java index 6ca8dc1037..1b62f6c820 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java @@ -82,7 +82,7 @@ public class QuartzScheduler implements SchedulerApi { try { JobDetail jobDetail; - //add a task (if this task already exists, return this task directly) + // add a task (if this task already exists, return this task directly) if (scheduler.checkExists(jobKey)) { jobDetail = scheduler.getJobDetail(jobKey); @@ -99,11 +99,9 @@ public class QuartzScheduler implements SchedulerApi { TriggerKey triggerKey = new TriggerKey(jobKey.getName(), jobKey.getGroup()); /* - * Instructs the Scheduler that upon a mis-fire - * situation, the CronTrigger wants to have it's - * next-fire-time updated to the next time in the schedule after the - * current time (taking into account any associated Calendar), - * but it does not want to be fired now. + * Instructs the Scheduler that upon a mis-fire situation, the CronTrigger wants to have it's next-fire-time + * updated to the next time in the schedule after the current time (taking into account any associated + * Calendar), but it does not want to be fired now. */ CronTrigger cronTrigger = newTrigger() .withIdentity(triggerKey) @@ -112,8 +110,7 @@ public class QuartzScheduler implements SchedulerApi { .withSchedule( cronSchedule(cronExpression) .withMisfireHandlingInstructionFireAndProceed() - .inTimeZone(DateUtils.getTimezone(timezoneId)) - ) + .inTimeZone(DateUtils.getTimezone(timezoneId))) .forJob(jobDetail).build(); if (scheduler.checkExists(triggerKey)) { @@ -124,12 +121,14 @@ public class QuartzScheduler implements SchedulerApi { if (!Strings.nullToEmpty(cronExpression).equalsIgnoreCase(Strings.nullToEmpty(oldCronExpression))) { // reschedule job trigger scheduler.rescheduleJob(triggerKey, cronTrigger); - logger.info("reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", + logger.info( + "reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", triggerKey.getName(), triggerKey.getGroup(), cronExpression, startDate, endDate); } } else { scheduler.scheduleJob(cronTrigger); - logger.info("schedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", + logger.info( + "schedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", triggerKey.getName(), triggerKey.getGroup(), cronExpression, startDate, endDate); } diff --git a/dolphinscheduler-scheduler-plugin/pom.xml b/dolphinscheduler-scheduler-plugin/pom.xml index 65b8873674..5865678eca 100644 --- a/dolphinscheduler-scheduler-plugin/pom.xml +++ b/dolphinscheduler-scheduler-plugin/pom.xml @@ -15,18 +15,17 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 - pom dolphinscheduler-scheduler-plugin + pom dolphinscheduler-scheduler-api @@ -44,4 +43,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java index 5b37b1f72d..50d0f835f7 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java @@ -38,7 +38,7 @@ public class SpringApplicationContext implements ApplicationContextAware, AutoCl */ @Override public void close() { - ((AbstractApplicationContext)applicationContext).close(); + ((AbstractApplicationContext) applicationContext).close(); } public static T getBean(Class requiredType) { diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java index 09c5571497..945efd574b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java @@ -20,5 +20,6 @@ package org.apache.dolphinscheduler.service.cache; import org.apache.dolphinscheduler.remote.command.Command; public interface CacheNotifyService { + void notifyMaster(Command command); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java index aa890af99d..f7ffe6ce16 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java @@ -173,7 +173,7 @@ public abstract class AbstractCycle { FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression(); return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always); } - + /** * whether the year field has a value of every or always * diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java index 81bf76a082..b15cee8648 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java @@ -17,12 +17,16 @@ package org.apache.dolphinscheduler.service.cron; -import com.cronutils.model.Cron; -import com.cronutils.model.definition.CronDefinitionBuilder; -import com.cronutils.model.time.ExecutionTime; -import com.cronutils.parser.CronParser; -import lombok.NonNull; -import org.apache.commons.collections.CollectionUtils; +import static com.cronutils.model.CronType.QUARTZ; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST; +import static org.apache.dolphinscheduler.common.Constants.COMMA; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.day; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.hour; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.min; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.month; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.week; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.year; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -30,8 +34,8 @@ import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.collections.CollectionUtils; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -44,15 +48,15 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import static com.cronutils.model.CronType.QUARTZ; -import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST; -import static org.apache.dolphinscheduler.common.Constants.COMMA; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.day; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.hour; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.min; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.month; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.week; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.year; +import lombok.NonNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.model.Cron; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.model.time.ExecutionTime; +import com.cronutils.parser.CronParser; /** * // todo: this utils is heavy, it rely on quartz and corn-utils. diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java index 00ca1029c6..b5c8bbba0f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java @@ -16,15 +16,17 @@ */ package org.apache.dolphinscheduler.service.cron; +import org.apache.dolphinscheduler.common.enums.CycleEnum; + import com.cronutils.model.Cron; import com.cronutils.model.field.expression.Always; import com.cronutils.model.field.expression.QuestionMark; -import org.apache.dolphinscheduler.common.enums.CycleEnum; /** * Crontab Cycle Tool Factory */ public class CycleFactory { + private CycleFactory() { throw new IllegalStateException("CycleFactory class"); } @@ -34,7 +36,7 @@ public class CycleFactory { * @return AbstractCycle */ public static AbstractCycle min(Cron cron) { - return new MinCycle(cron); + return new MinCycle(cron); } /** @@ -43,7 +45,7 @@ public class CycleFactory { * @return AbstractCycle */ public static AbstractCycle hour(Cron cron) { - return new HourCycle(cron); + return new HourCycle(cron); } /** @@ -52,7 +54,7 @@ public class CycleFactory { * @return AbstractCycle */ public static AbstractCycle day(Cron cron) { - return new DayCycle(cron); + return new DayCycle(cron); } /** @@ -61,7 +63,7 @@ public class CycleFactory { * @return AbstractCycle */ public static AbstractCycle week(Cron cron) { - return new WeekCycle(cron); + return new WeekCycle(cron); } /** @@ -70,9 +72,9 @@ public class CycleFactory { * @return AbstractCycle */ public static AbstractCycle month(Cron cron) { - return new MonthCycle(cron); + return new MonthCycle(cron); } - + /** * year * @param cron cron @@ -82,217 +84,219 @@ public class CycleFactory { return new YearCycle(cron); } - /** - * day cycle - */ - public static class DayCycle extends AbstractCycle { + /** + * day cycle + */ + public static class DayCycle extends AbstractCycle { - public DayCycle(Cron cron) { - super(cron); - } + public DayCycle(Cron cron) { + super(cron); + } - /** - * get cycle - * @return CycleEnum - */ + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - if (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthFieldIsEvery() - && dayOfWeekField.getExpression() instanceof QuestionMark - && monthField.getExpression() instanceof Always) { - return CycleEnum.DAY; - } + if (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthFieldIsEvery() + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthField.getExpression() instanceof Always) { + return CycleEnum.DAY; + } - return null; + return null; } - /** - * get min cycle - * @return CycleEnum - */ + /** + * get min cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if (dayOfMonthFieldIsEvery()) { - return CycleEnum.DAY; - } + if (dayOfMonthFieldIsEvery()) { + return CycleEnum.DAY; + } - return null; + return null; } - } + } + + /** + * hour cycle + */ + public static class HourCycle extends AbstractCycle { + + public HourCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getCycle() { + if (minFiledIsSetAll() + && hourFiledIsEvery() + && dayOfMonthField.getExpression() instanceof Always + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthField.getExpression() instanceof Always) { + return CycleEnum.HOUR; + } - /** - * hour cycle - */ - public static class HourCycle extends AbstractCycle { + return null; + } - public HourCycle(Cron cron) { - super(cron); + /** + * get mini cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getMiniCycle() { + if (hourFiledIsEvery()) { + return CycleEnum.HOUR; + } + return null; + } } - /** - * get cycle - * @return CycleEnum - */ + /** + * minute cycle + */ + public static class MinCycle extends AbstractCycle { + + public MinCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - if (minFiledIsSetAll() - && hourFiledIsEvery() - && dayOfMonthField.getExpression() instanceof Always - && dayOfWeekField.getExpression() instanceof QuestionMark - && monthField.getExpression() instanceof Always) { - return CycleEnum.HOUR; - } - - return null; + if (minFiledIsEvery() + && hourField.getExpression() instanceof Always + && dayOfMonthField.getExpression() instanceof Always + && monthField.getExpression() instanceof Always) { + return CycleEnum.MINUTE; + } + + return null; } - /** - * get mini cycle - * @return CycleEnum - */ + /** + * get min cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if(hourFiledIsEvery()){ - return CycleEnum.HOUR; - } - return null; + if (minFiledIsEvery()) { + return CycleEnum.MINUTE; + } + return null; } - } - - /** - * minute cycle - */ - public static class MinCycle extends AbstractCycle { - - public MinCycle(Cron cron) { - super(cron); - } - - /** - * get cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getCycle() { - if (minFiledIsEvery() - && hourField.getExpression() instanceof Always - && dayOfMonthField.getExpression() instanceof Always - && monthField.getExpression() instanceof Always) { - return CycleEnum.MINUTE; - } - - return null; - } - - /** - * get min cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getMiniCycle() { - if(minFiledIsEvery()){ - return CycleEnum.MINUTE; - } - return null; - } - } - - /** - * month cycle - */ - public static class MonthCycle extends AbstractCycle { - - public MonthCycle(Cron cron) { - super(cron); } - /** - * get cycle - * @return CycleEnum - */ + /** + * month cycle + */ + public static class MonthCycle extends AbstractCycle { + + public MonthCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - boolean flag = (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthFieldIsSetAll() - && dayOfWeekField.getExpression() instanceof QuestionMark - && monthFieldIsEvery()) || - (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthField.getExpression() instanceof QuestionMark - && dayofWeekFieldIsSetAll() - && monthFieldIsEvery()); - if (flag) { - return CycleEnum.MONTH; - } - - return null; + boolean flag = (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthFieldIsSetAll() + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthFieldIsEvery()) || + (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthField.getExpression() instanceof QuestionMark + && dayofWeekFieldIsSetAll() + && monthFieldIsEvery()); + if (flag) { + return CycleEnum.MONTH; + } + + return null; } - /** - * get mini cycle - * @return CycleEnum - */ + /** + * get mini cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if (monthFieldIsEvery()) { - return CycleEnum.MONTH; - } + if (monthFieldIsEvery()) { + return CycleEnum.MONTH; + } - return null; + return null; } - } - - /** - * week cycle - */ - public static class WeekCycle extends AbstractCycle { - public WeekCycle(Cron cron) { - super(cron); } - /** - * get cycle - * @return CycleEnum - */ + /** + * week cycle + */ + public static class WeekCycle extends AbstractCycle { + + public WeekCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - if (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthField.getExpression() instanceof QuestionMark - && dayofWeekFieldIsEvery() - && monthField.getExpression() instanceof Always) { - return CycleEnum.WEEK; - } - - return null; + if (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthField.getExpression() instanceof QuestionMark + && dayofWeekFieldIsEvery() + && monthField.getExpression() instanceof Always) { + return CycleEnum.WEEK; + } + + return null; } - /** - * get mini cycle - * @return CycleEnum - */ + /** + * get mini cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if (dayofWeekFieldIsEvery()) { - return CycleEnum.WEEK; - } + if (dayofWeekFieldIsEvery()) { + return CycleEnum.WEEK; + } - return null; + return null; } - } - + } + /** * year cycle */ public static class YearCycle extends AbstractCycle { + public YearCycle(Cron cron) { super(cron); } - + /** * get cycle * @return CycleEnum @@ -311,14 +315,14 @@ public class CycleFactory { && dayofWeekFieldIsSetAll() && monthFieldIsSetAll() && yearFieldIsEvery()); - + if (flag) { return CycleEnum.YEAR; } - + return null; } - + /** * get mini cycle * @return CycleEnum @@ -328,7 +332,7 @@ public class CycleFactory { if (yearFieldIsEvery()) { return CycleEnum.YEAR; } - + return null; } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java index d4011bcd74..20d0dcb5f2 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java @@ -16,62 +16,64 @@ */ package org.apache.dolphinscheduler.service.cron; -import com.cronutils.model.Cron; import org.apache.dolphinscheduler.common.enums.CycleEnum; import java.util.ArrayList; import java.util.List; +import com.cronutils.model.Cron; + /** * DAG Cycle judge */ public class CycleLinks extends AbstractCycle { - private final List cycleList = new ArrayList<>(); - public CycleLinks(Cron cron) { - super(cron); - } + private final List cycleList = new ArrayList<>(); - /** - * add cycle - * @param cycle cycle - * @return CycleLinks - */ - @Override - public CycleLinks addCycle(AbstractCycle cycle) { - cycleList.add(cycle); - return this; - } + public CycleLinks(Cron cron) { + super(cron); + } - /** - * get cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getCycle() { - for (AbstractCycle abstractCycle : cycleList) { - CycleEnum cycle = abstractCycle.getCycle(); - if (cycle != null) { - return cycle; - } + /** + * add cycle + * @param cycle cycle + * @return CycleLinks + */ + @Override + public CycleLinks addCycle(AbstractCycle cycle) { + cycleList.add(cycle); + return this; } - return null; - } + /** + * get cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getCycle() { + for (AbstractCycle abstractCycle : cycleList) { + CycleEnum cycle = abstractCycle.getCycle(); + if (cycle != null) { + return cycle; + } + } - /** - * get mini cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getMiniCycle() { - for (AbstractCycle cycleHelper : cycleList) { - CycleEnum cycle = cycleHelper.getMiniCycle(); - if (cycle != null) { - return cycle; - } + return null; } - return null; - } -} \ No newline at end of file + /** + * get mini cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getMiniCycle() { + for (AbstractCycle cycleHelper : cycleList) { + CycleEnum cycle = cycleHelper.getMiniCycle(); + if (cycle != null) { + return cycle; + } + } + + return null; + } +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java index 3342bd6bba..dfb4f14c95 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java @@ -17,11 +17,9 @@ package org.apache.dolphinscheduler.service.expand; -import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; @@ -29,6 +27,8 @@ import java.util.Date; import java.util.List; import java.util.Map; +import lombok.NonNull; + public interface CuringParamsService { /** @@ -65,7 +65,9 @@ public interface CuringParamsService { * @param timezone * @return */ - String curingGlobalParams(Integer processInstanceId, Map globalParamMap, List globalParamList, CommandType commandType, Date scheduleTime, String timezone); + String curingGlobalParams(Integer processInstanceId, Map globalParamMap, + List globalParamList, CommandType commandType, Date scheduleTime, + String timezone); /** * param parsing preparation @@ -74,7 +76,9 @@ public interface CuringParamsService { * @param processInstance * @return */ - Map paramParsingPreparation(@NonNull TaskInstance taskInstance, @NonNull AbstractParameters parameters, @NonNull ProcessInstance processInstance); + Map paramParsingPreparation(@NonNull TaskInstance taskInstance, + @NonNull AbstractParameters parameters, + @NonNull ProcessInstance processInstance); /** * preBuildBusinessParams diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java index c45665ab99..9324440970 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.SerializerProvider; @JsonComponent public class DateJsonSerializer extends JsonSerializer { + @Override public void serialize(Date value, JsonGenerator gen, SerializerProvider serializers) throws IOException { gen.writeString(DateUtils.dateToString(value)); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java index 73d420ccf0..e5405489e3 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java @@ -102,6 +102,7 @@ public class MasterPriorityQueue implements TaskPriorityQueue { * server comparator, used to sort server by createTime in reverse order. */ private class ServerComparator implements Comparator { + @Override public int compare(Server o1, Server o2) { return o2.getCreateTime().compareTo(o1.getCreateTime()); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java index 0cf03abe8d..30b6715082 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java @@ -36,6 +36,7 @@ import com.google.common.base.Preconditions; * All the task instances are in the same process instance. */ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue { + /** * queue size */ @@ -88,7 +89,8 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue { if (this.getTaskInstancePriority() < other.getTaskInstancePriority()) { return -1; } - if(this.getTaskGroupPriority() != other.getTaskGroupPriority()){ + if (this.getTaskGroupPriority() != other.getTaskGroupPriority()) { // larger number, higher priority - return Constants.OPPOSITE_VALUE * Integer.compare(this.getTaskGroupPriority(), other.getTaskGroupPriority()); + return Constants.OPPOSITE_VALUE + * Integer.compare(this.getTaskGroupPriority(), other.getTaskGroupPriority()); } if (this.getTaskId() > other.getTaskId()) { return 1; @@ -193,9 +194,11 @@ public class TaskPriority implements Comparable { if (this.getTaskId() < other.getTaskId()) { return -1; } - String thisGroupName = StringUtils.isNotBlank(this.getGroupName()) ? this.getGroupName() : Constants.EMPTY_STRING; - String otherGroupName = StringUtils.isNotBlank(other.getGroupName()) ? other.getGroupName() : Constants.EMPTY_STRING; - if(!thisGroupName.equals(otherGroupName)){ + String thisGroupName = + StringUtils.isNotBlank(this.getGroupName()) ? this.getGroupName() : Constants.EMPTY_STRING; + String otherGroupName = + StringUtils.isNotBlank(other.getGroupName()) ? other.getGroupName() : Constants.EMPTY_STRING; + if (!thisGroupName.equals(otherGroupName)) { return thisGroupName.compareTo(otherGroupName); } return Long.compare(this.getCheckpoint(), other.getCheckpoint()); @@ -211,45 +214,45 @@ public class TaskPriority implements Comparable { } TaskPriority that = (TaskPriority) o; return processInstancePriority == that.processInstancePriority - && processInstanceId == that.processInstanceId - && taskInstancePriority == that.taskInstancePriority - && taskId == that.taskId - && taskGroupPriority == that.taskGroupPriority - && Objects.equals(groupName, that.groupName); + && processInstanceId == that.processInstanceId + && taskInstancePriority == that.taskInstancePriority + && taskId == that.taskId + && taskGroupPriority == that.taskGroupPriority + && Objects.equals(groupName, that.groupName); } @Override public int hashCode() { return Objects.hash(processInstancePriority, - processInstanceId, - taskInstancePriority, - taskId, - taskGroupPriority, - groupName); + processInstanceId, + taskInstancePriority, + taskId, + taskGroupPriority, + groupName); } @Override public String toString() { return "TaskPriority{" - + "processInstancePriority=" - + processInstancePriority - + ", processInstanceId=" - + processInstanceId - + ", taskInstancePriority=" - + taskInstancePriority - + ", taskId=" - + taskId - + ", taskExecutionContext=" - + taskExecutionContext - + ", groupName='" - + groupName - + '\'' - + ", context=" - + context - + ", checkpoint=" - + checkpoint - + ", taskGroupPriority=" - + taskGroupPriority - + '}'; + + "processInstancePriority=" + + processInstancePriority + + ", processInstanceId=" + + processInstanceId + + ", taskInstancePriority=" + + taskInstancePriority + + ", taskId=" + + taskId + + ", taskExecutionContext=" + + taskExecutionContext + + ", groupName='" + + groupName + + '\'' + + ", context=" + + context + + ", checkpoint=" + + checkpoint + + ", taskGroupPriority=" + + taskGroupPriority + + '}'; } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java index 0007581d3c..736e117fe5 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java @@ -43,7 +43,6 @@ public interface TaskPriorityQueue { */ T take() throws TaskPriorityQueueException, InterruptedException; - /** * poll taskInfo with timeout * @param timeout @@ -61,4 +60,4 @@ public interface TaskPriorityQueue { * @throws TaskPriorityQueueException */ int size() throws TaskPriorityQueueException; -} \ No newline at end of file +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java index 0d0eebe03d..d9578b0757 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java @@ -68,7 +68,7 @@ public class TaskPriorityQueueImpl implements TaskPriorityQueue { */ @Override public TaskPriority poll(long timeout, TimeUnit unit) throws TaskPriorityQueueException, InterruptedException { - return queue.poll(timeout,unit); + return queue.poll(timeout, unit); } /** diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java index e3776e09ba..6bcd079870 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java @@ -22,9 +22,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import java.util.Collections; import java.util.HashMap; @@ -32,8 +29,13 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + @Component public class TaskPluginManager { + private static final Logger logger = LoggerFactory.getLogger(TaskPluginManager.class); private final Map taskChannelFactoryMap = new HashMap<>(); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java index 4481448a1d..f0ba597f50 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java @@ -16,17 +16,19 @@ */ package org.apache.dolphinscheduler.service.utils; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.service.model.TaskNode; + +import java.util.List; + import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import java.util.List; @Data @NoArgsConstructor public class ProcessData { + @EqualsAndHashCode.Include private List tasks; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java index 3b7d7b2eae..dd065cb0b6 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java @@ -24,6 +24,11 @@ import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseResult; import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -35,10 +40,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - @ExtendWith(MockitoExtension.class) public class AlertClientServiceTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java index a1622da903..2dc864eb1e 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java @@ -24,6 +24,11 @@ import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.TaskInstance; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -32,10 +37,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - /** * ProcessAlertManager Test */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java index 8655def27b..595e3cc071 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java @@ -28,6 +28,10 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.service.cache.impl.CacheNotifyServiceImpl; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,9 +40,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.ArrayList; -import java.util.List; - /** * tenant cache proxy test */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java index 199a351feb..6be6d95459 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java @@ -17,26 +17,36 @@ package org.apache.dolphinscheduler.service.cron; -import com.cronutils.builder.CronBuilder; -import com.cronutils.model.Cron; -import com.cronutils.model.CronType; -import com.cronutils.model.definition.CronDefinitionBuilder; -import com.cronutils.model.field.CronField; -import com.cronutils.model.field.CronFieldName; -import com.cronutils.model.field.expression.*; +import static com.cronutils.model.field.expression.FieldExpressionFactory.always; +import static com.cronutils.model.field.expression.FieldExpressionFactory.every; +import static com.cronutils.model.field.expression.FieldExpressionFactory.on; +import static com.cronutils.model.field.expression.FieldExpressionFactory.questionMark; + import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Date; -import static com.cronutils.model.field.expression.FieldExpressionFactory.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.builder.CronBuilder; +import com.cronutils.model.Cron; +import com.cronutils.model.CronType; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.model.field.CronField; +import com.cronutils.model.field.CronFieldName; +import com.cronutils.model.field.expression.Always; +import com.cronutils.model.field.expression.And; +import com.cronutils.model.field.expression.Between; +import com.cronutils.model.field.expression.Every; +import com.cronutils.model.field.expression.On; +import com.cronutils.model.field.expression.QuestionMark; /** * CronUtilsTest @@ -51,16 +61,15 @@ public class CronUtilsTest { @Test public void testCronAsString() { Cron cron = CronBuilder.cron(CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)).withYear(always()) - .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) - .withSecond(on(0)).instance(); + .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) + .withSecond(on(0)).instance(); // Obtain the string expression String cronAsString = cron.asString(); - // 0 */5 * * * ? * Every five minutes(once every 5 minutes) + // 0 */5 * * * ? * Every five minutes(once every 5 minutes) Assertions.assertEquals("0 */5 * * * ? *", cronAsString); } - /** * cron parse test */ @@ -112,12 +121,12 @@ public class CronUtilsTest { @Test public void test2() throws CronParseException { Cron cron1 = CronBuilder.cron(CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)).withYear(always()) - .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) - .withSecond(on(0)).instance(); + .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) + .withSecond(on(0)).instance(); // minute cycle String[] cronArayy = - new String[] {"* * * * * ? *", "* 0 * * * ? *", "* 5 * * 3/5 ? *", "0 0 * * * ? *", "0 0 7 * 1 ? *", - "0 0 7 * 1/1 ? *", "0 0 7 * 1-2 ? *", "0 0 7 * 1,2 ? *"}; + new String[]{"* * * * * ? *", "* 0 * * * ? *", "* 5 * * 3/5 ? *", "0 0 * * * ? *", "0 0 7 * 1 ? *", + "0 0 7 * 1/1 ? *", "0 0 7 * 1-2 ? *", "0 0 7 * 1,2 ? *"}; for (String minCrontab : cronArayy) { Cron cron = CronUtils.parse2Cron(minCrontab); CronField minField = cron.retrieve(CronFieldName.MINUTE); @@ -140,7 +149,8 @@ public class CronUtilsTest { logger.info("dayOfMonthField instanceof On:" + (dayOfMonthField.getExpression() instanceof On)); logger.info("dayOfMonthField instanceof And:" + (dayOfMonthField.getExpression() instanceof And)); logger.info( - "dayOfMonthField instanceof QuestionMark:" + (dayOfMonthField.getExpression() instanceof QuestionMark)); + "dayOfMonthField instanceof QuestionMark:" + + (dayOfMonthField.getExpression() instanceof QuestionMark)); CronField monthField = cron.retrieve(CronFieldName.MONTH); logger.info("monthField instanceof Between:" + (monthField.getExpression() instanceof Between)); @@ -157,7 +167,8 @@ public class CronUtilsTest { logger.info("dayOfWeekField instanceof On:" + (dayOfWeekField.getExpression() instanceof On)); logger.info("dayOfWeekField instanceof And:" + (dayOfWeekField.getExpression() instanceof And)); logger.info( - "dayOfWeekField instanceof QuestionMark:" + (dayOfWeekField.getExpression() instanceof QuestionMark)); + "dayOfWeekField instanceof QuestionMark:" + + (dayOfWeekField.getExpression() instanceof QuestionMark)); CronField yearField = cron.retrieve(CronFieldName.YEAR); logger.info("yearField instanceof Between:" + (yearField.getExpression() instanceof Between)); @@ -180,9 +191,11 @@ public class CronUtilsTest { @Test public void getSelfFireDateList() throws CronParseException { ZonedDateTime from = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:00:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:00:00").toInstant(), + ZoneId.systemDefault()); ZonedDateTime to = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-31 00:00:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-31 00:00:00").toInstant(), + ZoneId.systemDefault()); // test date Assertions.assertEquals(0, CronUtils.getFireDateList(to, from, "0 0 0 * * ? ").size()); try { @@ -197,20 +210,25 @@ public class CronUtilsTest { Assertions.assertEquals(30, CronUtils.getFireDateList(from, to, "0 0 0 * * ? ").size()); // test other Assertions.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2Cron("0 0 0 * * ? ")).size()); - Assertions.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2Cron("0 0 0 * * ? "), 5).size()); + Assertions.assertEquals(5, + CronUtils.getSelfFireDateList(from, to, CronUtils.parse2Cron("0 0 0 * * ? "), 5).size()); from = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), + ZoneId.systemDefault()); to = ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), ZoneId.systemDefault()); Assertions.assertEquals(1, - CronUtils.getFireDateList(from.minusSeconds(1L), to, CronUtils.parse2Cron("0 * * * * ? ")).size()); + CronUtils.getFireDateList(from.minusSeconds(1L), to, CronUtils.parse2Cron("0 * * * * ? ")).size()); from = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), + ZoneId.systemDefault()); to = ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:04:00").toInstant(), - ZoneId.systemDefault()); + ZoneId.systemDefault()); Assertions.assertEquals(2, - CronUtils.getFireDateList(from.minusSeconds(1L), to.minusSeconds(1L), CronUtils.parse2Cron("0 * * * * ? ")) - .size()); + CronUtils + .getFireDateList(from.minusSeconds(1L), to.minusSeconds(1L), + CronUtils.parse2Cron("0 * * * * ? ")) + .size()); } @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java index b8fe224561..464e1ea8d9 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.apache.dolphinscheduler.service.expand; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -24,6 +23,13 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,8 +39,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.*; - @ExtendWith(MockitoExtension.class) public class CuringGlobalParamsServiceTest { @@ -61,7 +65,8 @@ public class CuringGlobalParamsServiceTest { @Test public void testConvertParameterPlaceholders() { - Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap)).thenReturn("2022-06-26"); + Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap)) + .thenReturn("2022-06-26"); String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap); Assertions.assertNotNull(result); } @@ -80,36 +85,42 @@ public class CuringGlobalParamsServiceTest { @Test public void testCuringGlobalParams() { - //define globalMap + // define globalMap Map globalParamMap = new HashMap<>(); globalParamMap.put("globalParams1", "Params1"); - //define globalParamList + // define globalParamList List globalParamList = new ArrayList<>(); - //define scheduleTime + // define scheduleTime Date scheduleTime = DateUtils.stringToDate("2019-12-20 00:00:00"); - //test globalParamList is null - String result = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + // test globalParamList is null + String result = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertNull(result); - Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, null, CommandType.START_CURRENT_TASK_PROCESS, null, null)); - Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, null, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null)); + Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, null, + CommandType.START_CURRENT_TASK_PROCESS, null, null)); + Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, null, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null)); - //test globalParamList is not null + // test globalParamList is not null Property property = new Property("testGlobalParam", Direct.IN, DataType.VARCHAR, "testGlobalParam"); globalParamList.add(property); - String result2 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result2 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertEquals(result2, JSONUtils.toJsonString(globalParamList)); - String result3 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, null, null); + String result3 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, null, null); Assertions.assertEquals(result3, JSONUtils.toJsonString(globalParamList)); - String result4 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result4 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertEquals(result4, JSONUtils.toJsonString(globalParamList)); - //test var $ startsWith + // test var $ startsWith globalParamMap.put("bizDate", "${system.biz.date}"); globalParamMap.put("b1zCurdate", "${system.biz.curdate}"); @@ -121,12 +132,14 @@ public class CuringGlobalParamsServiceTest { globalParamList.add(property3); globalParamList.add(property4); - String result5 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result5 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertEquals(result5, JSONUtils.toJsonString(globalParamList)); Property testStartParamProperty = new Property("testStartParam", Direct.IN, DataType.VARCHAR, ""); globalParamList.add(testStartParamProperty); - Property testStartParam2Property = new Property("testStartParam2", Direct.IN, DataType.VARCHAR, "$[yyyy-MM-dd+1]"); + Property testStartParam2Property = + new Property("testStartParam2", Direct.IN, DataType.VARCHAR, "$[yyyy-MM-dd+1]"); globalParamList.add(testStartParam2Property); globalParamMap.put("testStartParam", ""); globalParamMap.put("testStartParam2", "$[yyyy-MM-dd+1]"); @@ -141,7 +154,8 @@ public class CuringGlobalParamsServiceTest { } } - String result6 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result6 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertTrue(result6.contains("20191220")); } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java index 025ed9f7cf..07e44205ee 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.service.expand; import org.apache.commons.lang3.StringUtils; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,7 +46,8 @@ public class TimePlaceholderResolverExpandServiceTest { boolean implCheckResult = timePlaceholderResolverExpandServiceImpl.timeFunctionNeedExpand(placeHolderName); Assertions.assertFalse(implCheckResult); - String implResultString = timePlaceholderResolverExpandServiceImpl.timeFunctionExtension(1, "", placeHolderName); + String implResultString = + timePlaceholderResolverExpandServiceImpl.timeFunctionExtension(1, "", placeHolderName); Assertions.assertTrue(StringUtils.isEmpty(implResultString)); } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java index 1c48368211..9ccfadd49d 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java @@ -28,6 +28,9 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import java.nio.charset.StandardCharsets; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -35,8 +38,6 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.nio.charset.StandardCharsets; - @ExtendWith(MockitoExtension.class) public class LogClientTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java index 44b783a7be..8bfd721709 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.service.log; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +32,8 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import io.netty.channel.Channel; + @ExtendWith(MockitoExtension.class) public class LoggerRequestProcessorTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java index ecc28cd8aa..19af256103 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; -import ch.qos.logback.core.spi.FilterReply; import org.apache.dolphinscheduler.common.Constants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.spi.FilterReply; + public class MasterLogFilterTest { @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java index d4514ab95c..ec99b610c9 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java @@ -17,16 +17,17 @@ package org.apache.dolphinscheduler.service.log; +import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler; + import org.apache.dolphinscheduler.common.Constants; + +import java.util.regex.Pattern; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.regex.Pattern; - -import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler; - public class SensitiveDataConverterTest { private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java index c8bec75d08..e06041d605 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; + public class TaskLogDiscriminatorTest { /** diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java index 56a1023fa4..9c1c9f80c9 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; -import ch.qos.logback.core.spi.FilterReply; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.spi.FilterReply; + public class TaskLogFilterTest { @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java index d0d408f797..c4f2c2f9a1 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; -import ch.qos.logback.core.spi.FilterReply; import org.apache.dolphinscheduler.common.Constants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.spi.FilterReply; + public class WorkerLogFilterTest { @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index bb47c74529..d0ef8f8e18 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -17,18 +17,65 @@ package org.apache.dolphinscheduler.service.process; -import com.fasterxml.jackson.databind.JsonNode; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE; +import static org.mockito.ArgumentMatchers.any; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum; +import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.DqExecuteResult; +import org.apache.dolphinscheduler.dao.entity.DqRule; +import org.apache.dolphinscheduler.dao.entity.DqRuleExecuteSql; +import org.apache.dolphinscheduler.dao.entity.DqRuleInputEntry; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; +import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; +import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.DqComparisonTypeMapper; +import org.apache.dolphinscheduler.dao.mapper.DqExecuteResultMapper; +import org.apache.dolphinscheduler.dao.mapper.DqRuleExecuteSqlMapper; +import org.apache.dolphinscheduler.dao.mapper.DqRuleInputEntryMapper; +import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.plugin.task.api.enums.dp.*; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.DqTaskState; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.InputType; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.OptionSourceType; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ValueType; import org.apache.dolphinscheduler.plugin.task.api.model.DateInterval; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.service.cron.CronUtilsTest; @@ -38,6 +85,14 @@ import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.base.FormType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -50,10 +105,7 @@ import org.mockito.quality.Strictness; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.*; -import static org.mockito.ArgumentMatchers.any; +import com.fasterxml.jackson.databind.JsonNode; /** * process service test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java index 3abe0db486..a430ab3c95 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java @@ -20,11 +20,12 @@ package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.service.exceptions.TaskPriorityQueueException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class PeerTaskInstancePriorityQueueTest { @Test @@ -47,7 +48,6 @@ public class PeerTaskInstancePriorityQueueTest { Assertions.assertTrue(queue.size() < peekBeforeLength); } - @Test public void poll() throws Exception { PeerTaskInstancePriorityQueue queue = getPeerTaskInstancePriorityQueue(); @@ -64,7 +64,7 @@ public class PeerTaskInstancePriorityQueueTest { } @Test - public void peekTaskGroupPriority() throws Exception{ + public void peekTaskGroupPriority() throws Exception { PeerTaskInstancePriorityQueue queue = new PeerTaskInstancePriorityQueue(); TaskInstance taskInstanceHigPriority = createTaskInstance("high", Priority.HIGH, 2); TaskInstance taskInstanceMediumPriority = createTaskInstance("medium", Priority.HIGH, 1); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java index a78cc959a0..aeaf079c9e 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java @@ -18,14 +18,15 @@ package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.enums.Priority; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class TaskPriorityQueueImplTest { @Test @@ -36,9 +37,8 @@ public class TaskPriorityQueueImplTest { List taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 1, 0, 0, 1, "default"); priorityTwo = new TaskPriority(0, 2, 0, 0, 1, "default"); @@ -46,9 +46,8 @@ public class TaskPriorityQueueImplTest { taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 1, 0, 1, "default"); priorityTwo = new TaskPriority(0, 0, 2, 0, 1, "default"); @@ -56,9 +55,8 @@ public class TaskPriorityQueueImplTest { taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 1, 1, "default"); priorityTwo = new TaskPriority(0, 0, 0, 2, 1, "default"); @@ -66,49 +64,44 @@ public class TaskPriorityQueueImplTest { taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 0, 1, "default_1"); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_2"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_2"); priorityThree = new TaskPriority(0, 0, 0, 0, 1, "default_3"); taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 0, 2, "default_1"); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_2"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_2"); priorityThree = new TaskPriority(0, 0, 0, 0, 3, "default_3"); taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( Arrays.asList(priorityThree, priorityOne, priorityTwo), - taskPriorities - ); + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 0, 1, "default_1"); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_2"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_2"); priorityThree = new TaskPriority(0, 0, 0, 0, 3, "default_3"); taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( Arrays.asList(priorityThree, priorityOne, priorityTwo), - taskPriorities - ); + taskPriorities); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_1"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_1"); priorityOne = new TaskPriority(0, 0, 0, 0, 1, "default_1"); priorityThree = new TaskPriority(0, 0, 0, 0, 3, "default_1"); taskPriorities = Arrays.asList(priorityTwo, priorityOne, priorityThree); Collections.sort(taskPriorities); Assertions.assertEquals( Arrays.asList(priorityThree, priorityTwo, priorityOne), - taskPriorities - ); + taskPriorities); } @Test @@ -166,4 +159,4 @@ public class TaskPriorityQueueImplTest { TaskPriority priorityOne = new TaskPriority(priority, processInstanceId, 0, 0, 1, "default"); return priorityOne; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java index d9d70172c5..a7398d7953 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.service.storage.impl; import org.apache.dolphinscheduler.common.utils.HttpUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java index 384a0139d1..68df14d990 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java @@ -17,8 +17,20 @@ package org.apache.dolphinscheduler.service.storage.impl; -import com.aliyun.oss.OSS; +import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; +import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import java.io.IOException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,13 +38,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.IOException; - -import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; +import com.aliyun.oss.OSS; @ExtendWith(MockitoExtension.class) public class OssOperatorTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java index 98ca251fd6..a53b3e0699 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java @@ -19,6 +19,10 @@ package org.apache.dolphinscheduler.service.utils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; + +import java.net.InetAddress; +import java.net.UnknownHostException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,9 +30,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.UnknownHostException; - /** * configuration test */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java index 4635e9b4ef..dcd2a734f0 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java @@ -17,12 +17,15 @@ package org.apache.dolphinscheduler.service.utils; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.sift.SiftingAppender; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.service.log.TaskLogDiscriminator; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -30,9 +33,8 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.LoggerFactory; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Date; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.sift.SiftingAppender; @ExtendWith(MockitoExtension.class) public class LogUtilsTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java index bec60332da..9c74bc3844 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java @@ -17,11 +17,17 @@ package org.apache.dolphinscheduler.service.utils; +import static org.mockito.ArgumentMatchers.anyString; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; + +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,11 +39,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.ArgumentMatchers.anyString; - @ExtendWith(MockitoExtension.class) public class ProcessUtilsTest { diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java index eceb39fda9..d423ae3a51 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; public interface DataSourceChannelFactory extends PrioritySPI { + /** * get datasource client */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java index 4bc2c1a2f6..9eef16d77e 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java @@ -72,7 +72,7 @@ public enum CommandType { static { for (CommandType commandType : CommandType.values()) { - COMMAND_TYPE_MAP.put(commandType.code,commandType); + COMMAND_TYPE_MAP.put(commandType.code, commandType); } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 99c5ec8187..cd00bf1234 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -27,6 +27,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; import com.google.common.base.Functions; public enum DbType { + MYSQL(0, "mysql"), POSTGRESQL(1, "postgresql"), HIVE(2, "hive"), @@ -37,8 +38,8 @@ public enum DbType { DB2(7, "db2"), PRESTO(8, "presto"), H2(9, "h2"), - REDSHIFT(10,"redshift"), - ATHENA(11,"athena"), + REDSHIFT(10, "redshift"), + ATHENA(11, "athena"), ; @EnumValue @@ -69,7 +70,8 @@ public enum DbType { } public static DbType ofName(String name) { - return Arrays.stream(DbType.values()).filter(e -> e.name().equals(name)).findFirst().orElseThrow(() -> new NoSuchElementException("no such db type")); + return Arrays.stream(DbType.values()).filter(e -> e.name().equals(name)).findFirst() + .orElseThrow(() -> new NoSuchElementException("no such db type")); } public boolean isHive() { diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java index 0640259e0d..242b3cdc42 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java @@ -26,6 +26,7 @@ package org.apache.dolphinscheduler.spi.enums; * have_alert */ public enum Flag { + /** * 0 no * 1 yes diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java index 8e80802b85..a072f36ec4 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * resource type */ public enum ResourceType { + /** * 0 file, 1 udf */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java index c5706d2b89..76fb5487e1 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java @@ -73,12 +73,16 @@ public class PluginParamsTransfer { * @param pluginParamsTemplate pluginParamsTemplate * @return return plugin params value */ - public static List> generatePluginParams(Map paramsMap, String pluginParamsTemplate) { + public static List> generatePluginParams(Map paramsMap, + String pluginParamsTemplate) { if (paramsMap == null || paramsMap.isEmpty()) { return null; } - List> pluginParamsList = JSONUtils.parseObject(pluginParamsTemplate, new TypeReference>>() {}); - pluginParamsList.forEach(pluginParams -> pluginParams.put(STRING_PLUGIN_PARAM_VALUE, paramsMap.get(pluginParams.get(STRING_PLUGIN_PARAM_FIELD)))); + List> pluginParamsList = + JSONUtils.parseObject(pluginParamsTemplate, new TypeReference>>() { + }); + pluginParamsList.forEach(pluginParams -> pluginParams.put(STRING_PLUGIN_PARAM_VALUE, + paramsMap.get(pluginParams.get(STRING_PLUGIN_PARAM_FIELD)))); return pluginParamsList; } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java index 91d493637c..cb020ab311 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java @@ -34,7 +34,8 @@ public class ParamsOptions { */ private boolean disabled; - public ParamsOptions() {} + public ParamsOptions() { + } public ParamsOptions(String label, Object value, boolean disabled) { this.label = label; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java index 1b65133c94..a8b1d4f95e 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.spi.params.base; +import static java.util.Objects.requireNonNull; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_EMIT; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_FIELD; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_NAME; @@ -26,8 +27,6 @@ import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARA import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_VALIDATE; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_VALUE; -import static java.util.Objects.requireNonNull; - import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; @@ -118,7 +117,8 @@ public class PluginParams { @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "set") public static class Builder { - //Must have + + // Must have protected String name; protected FormType formType; @@ -127,7 +127,7 @@ public class PluginParams { protected String fieldName; - //option params + // option params protected ParamsProps props; protected Object value; @@ -154,7 +154,7 @@ public class PluginParams { this.fieldName = title; } - //for json deserialize to POJO + // for json deserialize to POJO @JsonCreator public Builder(@JsonProperty("field") String name, @JsonProperty("type") FormType formType, @@ -166,8 +166,7 @@ public class PluginParams { @JsonProperty("emit") List emit, @JsonProperty("info") String info, @JsonProperty("hidden") Boolean hidden, - @JsonProperty("display") Boolean display - ) { + @JsonProperty("display") Boolean display) { requireNonNull(name, "name is null"); requireNonNull(formType, "formType is null"); requireNonNull(title, "title is null"); @@ -222,5 +221,3 @@ public class PluginParams { } } - - diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java index d26ffac666..cc02e9581f 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java @@ -64,6 +64,7 @@ public class Validate { @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "set") public static class Builder { + private boolean required = false; private String message; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java index b6bfe17cf9..a06bdc26f0 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java @@ -33,7 +33,7 @@ public class SwitchParam extends PluginParams { private SwitchParam(Builder builder) { super(builder); } - + private SwitchParamProps props; public static Builder newBuilder(String name, String title) { diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java index 3268b6a4ee..c2a6de5cdd 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java @@ -75,7 +75,7 @@ public class GroupParam extends PluginParams { this.emit = emit; return this; } - + public Builder setProps(GroupParamsProps props) { this.props = props; return this; @@ -86,10 +86,10 @@ public class GroupParam extends PluginParams { this.setProps(new GroupParamsProps()); } - ((GroupParamsProps)this.props).setRules(rules); + ((GroupParamsProps) this.props).setRules(rules); return this; } - + @Override public GroupParam build() { return new GroupParam(this); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java index 2a16bd933e..0c838edf63 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java @@ -53,7 +53,7 @@ public class InputParam extends PluginParams { this.props = props; return this; } - + public Builder setPlaceholder(String placeholder) { if (this.props == null) { this.setProps(new InputParamProps()); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java index 7ba34d94d2..cd4c527c5d 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java @@ -53,11 +53,11 @@ public class RadioParam extends PluginParams { public Builder(String name, String title) { super(name, RADIO, title); } - + private List options; private RadioParamProps props; - + public Builder setOptions(List options) { this.options = options; return this; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java index 8b1921b700..970e7cd66d 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.spi.plugin; -import lombok.extern.slf4j.Slf4j; - import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class PrioritySPIFactory { @@ -48,8 +48,9 @@ public class PrioritySPIFactory { T oldSPI = map.get(identify.getName()); if (newSPI.compareTo(oldSPI.getIdentify().getPriority()) == 0) { - throw new IllegalArgumentException(String.format("These two spi plugins has conflict identify name with the same priority: %s, %s", - oldSPI.getIdentify(), newSPI.getIdentify())); + throw new IllegalArgumentException( + String.format("These two spi plugins has conflict identify name with the same priority: %s, %s", + oldSPI.getIdentify(), newSPI.getIdentify())); } else if (newSPI.compareTo(oldSPI.getIdentify().getPriority()) > 0) { log.info("The {} plugin has high priority, will override {}", newSPI.getIdentify(), oldSPI); map.put(identify.getName(), newSPI); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index 259bb45322..7762bc1180 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -21,6 +21,7 @@ package org.apache.dolphinscheduler.spi.utils; * constants */ public class Constants { + private Constants() { throw new IllegalStateException("Constants class"); } @@ -44,7 +45,6 @@ public class Constants { /**plugin param emit string **/ public static final String STRING_PLUGIN_PARAM_EMIT = "emit"; - /** string true */ public static final String STRING_TRUE = "true"; /** string false */ @@ -122,7 +122,8 @@ public class Constants { /** * hadoop.security.authentication */ - public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = + "hadoop.security.authentication.startup.state"; /** * loginUserFromKeytab user @@ -164,7 +165,6 @@ public class Constants { public static final String COM_REDSHIFT_JDBC_DRIVER = "com.amazon.redshift.jdbc42.Driver"; public static final String COM_ATHENA_JDBC_DRIVER = "com.simba.athena.jdbc.Driver"; - /** * validation Query */ @@ -224,7 +224,6 @@ public class Constants { */ public static final String COLON = ":"; - /** * AT SIGN @ */ @@ -235,7 +234,6 @@ public class Constants { */ public static final String SEMICOLON = ";"; - /** * EQUAL_SIGN = */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java index 4bc9504693..58877f3e97 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java @@ -27,6 +27,7 @@ import java.util.Objects; import java.util.TimeZone; import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -441,8 +442,7 @@ public class DateUtils { * @param timeStamp time stamp (milliseconds) * @return local date */ - public static @Nullable - Date timeStampToDate(long timeStamp) { + public static @Nullable Date timeStampToDate(long timeStamp) { return timeStamp <= 0L ? null : new Date(timeStamp); } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java index 5fa432a0ff..cce60c30b3 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.spi.utils; -import static java.nio.charset.StandardCharsets.UTF_8; - import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.text.SimpleDateFormat; @@ -34,21 +33,21 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.module.SimpleModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; @@ -62,7 +61,7 @@ public class JSONUtils { private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); static { - logger.info("init timezone: {}",TimeZone.getDefault()); + logger.info("init timezone: {}", TimeZone.getDefault()); } private static final SimpleModule LOCAL_DATE_TIME_MODULE = new SimpleModule() @@ -190,7 +189,8 @@ public class JSONUtils { * @return json to map */ public static Map toMap(String json) { - return parseObject(json, new TypeReference>() {}); + return parseObject(json, new TypeReference>() { + }); } /** @@ -204,7 +204,8 @@ public class JSONUtils { * @return to map */ public static Map toMap(String json, Class classK, Class classV) { - return parseObject(json, new TypeReference>() {}); + return parseObject(json, new TypeReference>() { + }); } /** @@ -250,7 +251,7 @@ public class JSONUtils { * @param object type * @return byte array */ - public static byte[] toJsonByteArray(T obj) { + public static byte[] toJsonByteArray(T obj) { if (obj == null) { return null; } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java index a98e38b59a..44891af647 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java @@ -216,27 +216,27 @@ public class StringUtils { out.write("\\u00" + hex(ch)); } else if (ch < 32) { switch (ch) { - case '\b' : + case '\b': out.write('\\'); out.write('b'); break; - case '\n' : + case '\n': out.write('\\'); out.write('n'); break; - case '\t' : + case '\t': out.write('\\'); out.write('t'); break; - case '\f' : + case '\f': out.write('\\'); out.write('f'); break; - case '\r' : + case '\r': out.write('\\'); out.write('r'); break; - default : + default: if (ch > 0xf) { out.write("\\u00" + hex(ch)); } else { @@ -246,27 +246,27 @@ public class StringUtils { } } else { switch (ch) { - case '\'' : + case '\'': if (escapeSingleQuote) { out.write('\\'); } out.write('\''); break; - case '"' : + case '"': out.write('\\'); out.write('"'); break; - case '\\' : + case '\\': out.write('\\'); out.write('\\'); break; - case '/' : + case '/': if (escapeForwardSlash) { out.write('\\'); } out.write('/'); break; - default : + default: out.write(ch); break; } @@ -282,10 +282,11 @@ public class StringUtils { mainParameter = mainParameter .replace(Constants.DOUBLE_BRACKETS_LEFT, Constants.DOUBLE_BRACKETS_LEFT_SPACE) .replace(Constants.DOUBLE_BRACKETS_RIGHT, Constants.DOUBLE_BRACKETS_RIGHT_SPACE); - if (mainParameter.contains(Constants.DOUBLE_BRACKETS_LEFT) || mainParameter.contains(Constants.DOUBLE_BRACKETS_RIGHT)) { + if (mainParameter.contains(Constants.DOUBLE_BRACKETS_LEFT) + || mainParameter.contains(Constants.DOUBLE_BRACKETS_RIGHT)) { return replaceDoubleBrackets(mainParameter); } else { - return mainParameter; + return mainParameter; } } } diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java index 1bc3c3157a..1f88584a83 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java @@ -23,14 +23,15 @@ import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.input.InputParam; import org.apache.dolphinscheduler.spi.params.radio.RadioParam; + +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.List; - /** * PluginParamsTransfer Tester. */ @@ -182,36 +183,38 @@ public class PluginParamsTransferTest { @Test public void testGetPluginParams() { - String paramsJsonAssert = "[{\"props\":null,\"field\":\"field1\",\"props\":null,\"type\":\"input\",\"title\":\"field1\",\"value\":\"v1\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field2\",\"props\":null,\"type\":\"input\",\"title\":\"field2\",\"value\":\"v2\",\"validate\":null}," - + "{\"field\":\"field3\",\"props\":null,\"type\":\"input\",\"title\":\"field3\",\"value\":\"v3\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field4\",\"props\":null,\"type\":\"input\",\"title\":\"field4\",\"value\":\"v4\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"number\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field5\",\"props\":null,\"type\":\"input\",\"title\":\"field5\",\"value\":\"v5\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field6\",\"props\":null,\"type\":\"radio\",\"title\":\"field6\",\"value\":true,\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" - + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," - + "{\"field\":\"field7\",\"props\":{\"type\":null,\"placeholder\":\"if enable use authentication, you need input user\",\"rows\":0}," - + "\"type\":\"input\",\"title\":\"field7\",\"value\":\"v6\",\"validate\":null},{\"field\":\"field8\",\"props\":{" - + "\"type\":\"PASSWORD\",\"placeholder\":\"if enable use authentication, you need input password\",\"rows\":0}," - + "\"type\":\"input\",\"title\":\"field8\",\"value\":\"v7\",\"validate\":null},{\"field\":\"field9\"," - + "\"props\":null,\"type\":\"radio\",\"title\":\"field9\",\"value\":false,\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" - + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," - + "{\"field\":\"field10\",\"props\":null,\"type\":\"radio\",\"title\":\"field10\",\"value\":false,\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]," - + "\"options\":[{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," - + "{\"field\":\"field11\",\"props\":null,\"type\":\"input\",\"title\":\"field11\",\"value\":\"*\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"showType\",\"props\":null,\"type\":\"radio\",\"title\":\"showType\",\"value\":\"table\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" - + "{\"label\":\"table\",\"value\":\"table\",\"disabled\":false},{\"label\":\"text\",\"value\":\"text\",\"disabled\":false}," - + "{\"label\":\"attachment\",\"value\":\"attachment\",\"disabled\":false},{\"label\":\"tableattachment\",\"value\":\"tableattachment\",\"disabled\":false}]}]"; + String paramsJsonAssert = + "[{\"props\":null,\"field\":\"field1\",\"props\":null,\"type\":\"input\",\"title\":\"field1\",\"value\":\"v1\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field2\",\"props\":null,\"type\":\"input\",\"title\":\"field2\",\"value\":\"v2\",\"validate\":null}," + + "{\"field\":\"field3\",\"props\":null,\"type\":\"input\",\"title\":\"field3\",\"value\":\"v3\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field4\",\"props\":null,\"type\":\"input\",\"title\":\"field4\",\"value\":\"v4\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"number\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field5\",\"props\":null,\"type\":\"input\",\"title\":\"field5\",\"value\":\"v5\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field6\",\"props\":null,\"type\":\"radio\",\"title\":\"field6\",\"value\":true,\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" + + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," + + "{\"field\":\"field7\",\"props\":{\"type\":null,\"placeholder\":\"if enable use authentication, you need input user\",\"rows\":0}," + + "\"type\":\"input\",\"title\":\"field7\",\"value\":\"v6\",\"validate\":null},{\"field\":\"field8\",\"props\":{" + + "\"type\":\"PASSWORD\",\"placeholder\":\"if enable use authentication, you need input password\",\"rows\":0}," + + "\"type\":\"input\",\"title\":\"field8\",\"value\":\"v7\",\"validate\":null},{\"field\":\"field9\"," + + "\"props\":null,\"type\":\"radio\",\"title\":\"field9\",\"value\":false,\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" + + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," + + "{\"field\":\"field10\",\"props\":null,\"type\":\"radio\",\"title\":\"field10\",\"value\":false,\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]," + + "\"options\":[{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," + + "{\"field\":\"field11\",\"props\":null,\"type\":\"input\",\"title\":\"field11\",\"value\":\"*\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"showType\",\"props\":null,\"type\":\"radio\",\"title\":\"showType\",\"value\":\"table\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" + + "{\"label\":\"table\",\"value\":\"table\",\"disabled\":false},{\"label\":\"text\",\"value\":\"text\",\"disabled\":false}," + + "{\"label\":\"attachment\",\"value\":\"attachment\",\"disabled\":false},{\"label\":\"tableattachment\",\"value\":\"tableattachment\",\"disabled\":false}]}]"; List pluginParams = PluginParamsTransfer.transferJsonToParamsList(paramsJsonAssert); - String[] results = new String[]{"v1", "v2", "v3", "v4", "v5", "true", "v6", "v7", "false", "false", "*", "table", "v1"}; + String[] results = + new String[]{"v1", "v2", "v3", "v4", "v5", "true", "v6", "v7", "false", "false", "*", "table", "v1"}; Assertions.assertEquals(12, pluginParams.size()); for (int i = 0; i < pluginParams.size(); i++) { PluginParams param = pluginParams.get(i); diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java index 71d3b8c868..64dedaa4b2 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java @@ -17,17 +17,19 @@ package org.apache.dolphinscheduler.spi.plugin; -import com.google.auto.service.AutoService; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Map; +import com.google.auto.service.AutoService; public class PrioritySPIFactoryTest { @Test public void loadHighPriority() { - PrioritySPIFactory factory = new PrioritySPIFactory<>(LoadHighPriorityConflictTestSPI.class); + PrioritySPIFactory factory = + new PrioritySPIFactory<>(LoadHighPriorityConflictTestSPI.class); Map spiMap = factory.getSPIMap(); Assertions.assertEquals(1, spiMap.get("A").getIdentify().getPriority()); } @@ -39,7 +41,6 @@ public class PrioritySPIFactoryTest { }); } - public interface LoadHighPriorityConflictTestSPI extends PrioritySPI { } @@ -84,5 +85,4 @@ public class PrioritySPIFactoryTest { } } - -} \ No newline at end of file +} diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java index b06e3a3237..09781d5ccf 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java @@ -17,20 +17,26 @@ package org.apache.dolphinscheduler.spi.utils; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.*; public class JSONUtilsTest { - @Test public void createObjectNodeTest() { String jsonStr = "{\"a\":\"b\",\"b\":\"d\"}"; @@ -53,7 +59,6 @@ public class JSONUtilsTest { } - @Test public void string2MapTest() { String str = list2String(); @@ -90,7 +95,6 @@ public class JSONUtilsTest { Assertions.assertNull(JSONUtils.parseObject("foo", String.class)); } - @Test public void testJsonByteArray() { String str = "foo"; @@ -182,7 +186,9 @@ public class JSONUtilsTest { String time = "2022-02-22 13:38:24"; Date date = DateUtils.stringToDate(time); LocalDateTime localDateTime = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - List timeList = JSONUtils.parseObject("[\"2022-02-22 13:38:24\"]", new TypeReference>(){}); + List timeList = + JSONUtils.parseObject("[\"2022-02-22 13:38:24\"]", new TypeReference>() { + }); Assertions.assertNotNull(timeList); Assertions.assertEquals(1, timeList.size()); Assertions.assertEquals(localDateTime, timeList.get(0)); diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java index 728fed8eaa..1a95ff53ce 100644 --- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java +++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java @@ -21,6 +21,8 @@ import org.apache.curator.test.TestingServer; import java.io.IOException; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; @@ -30,8 +32,6 @@ import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; -import lombok.NonNull; - @SpringBootApplication public class StandaloneServer implements ApplicationListener { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml index eb18a84201..256aa99144 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml @@ -225,7 +225,7 @@ ${project.version} - + org.apache.dolphinscheduler dolphinscheduler-task-datasync ${project.version} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java index 72d2dcc8f7..98e33761b6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java @@ -148,7 +148,7 @@ public abstract class AbstractShell { timeOutTimer = new Timer(); timeoutTimerTask = new ShellTimeoutTimerTask( this); - //One time scheduling. + // One time scheduling. timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); } final BufferedReader errReader = @@ -162,6 +162,7 @@ public abstract class AbstractShell { // read error and input streams as this would free up the buffers // free the error stream buffer Thread errThread = new Thread() { + @Override public void run() { try { @@ -177,6 +178,7 @@ public abstract class AbstractShell { } }; Thread inThread = new Thread() { + @Override public void run() { try { @@ -204,8 +206,8 @@ public abstract class AbstractShell { logger.warn("Interrupted while reading the error and in stream", ie); } completed.compareAndSet(false, true); - //the timeout thread handling - //taken care in finally block + // the timeout thread handling + // taken care in finally block if (exitCode != 0 || errMsg.length() > 0) { throw new ExitCodeException(exitCode, errMsg.toString()); } @@ -246,8 +248,7 @@ public abstract class AbstractShell { * @param lines lines * @throws IOException errors */ - protected abstract void parseExecResult(BufferedReader lines) - throws IOException; + protected abstract void parseExecResult(BufferedReader lines) throws IOException; /** * get the current sub-process executing the given command @@ -291,9 +292,9 @@ public abstract class AbstractShell { try { p.exitValue(); } catch (Exception e) { - //Process has not terminated. - //So check if it has completed - //if not just destroy it. + // Process has not terminated. + // So check if it has completed + // if not just destroy it. if (p != null && !shell.completed.get()) { shell.setTimedOut(); p.destroy(); @@ -306,6 +307,7 @@ public abstract class AbstractShell { * This is an IOException with exit code added. */ public static class ExitCodeException extends IOException { + private final int exitCode; public ExitCodeException(int exitCode, String message) { @@ -322,6 +324,7 @@ public abstract class AbstractShell { * process manage container */ public static class ProcessContainer extends ConcurrentHashMap { + private static final ProcessContainer container = new ProcessContainer(); private ProcessContainer() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java index 4916d8e57f..7e39f184fa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java @@ -24,6 +24,7 @@ import java.io.Serializable; */ public class K8sTaskExecutionContext implements Serializable { + private String configYaml; public String getConfigYaml() { @@ -37,7 +38,7 @@ public class K8sTaskExecutionContext implements Serializable { @Override public String toString() { return "K8sTaskExecutionContext{" - + "configYaml='" + configYaml + '\'' - + '}'; + + "configYaml='" + configYaml + '\'' + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java index 91be73620b..22518f7553 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java @@ -24,12 +24,13 @@ import org.apache.commons.lang3.SystemUtils; import java.util.regex.Matcher; import java.util.regex.Pattern; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import lombok.NonNull; - public final class ProcessUtils { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); private ProcessUtils() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java index 9c025e800c..e1d6432999 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java @@ -27,7 +27,6 @@ import java.util.List; */ public class SQLTaskExecutionContext implements Serializable { - /** * warningGroupId */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java index 9ccc438277..e1a4fc4a9f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java @@ -67,10 +67,8 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { @Override protected String buildCommandFilePath() { // command file - return String.format("%s/%s.%s" - , taskRequest.getExecutePath() - , taskRequest.getTaskAppId() - , SystemUtils.IS_OS_WINDOWS ? "bat" : "command"); + return String.format("%s/%s.%s", taskRequest.getExecutePath(), taskRequest.getTaskAppId(), + SystemUtils.IS_OS_WINDOWS ? "bat" : "command"); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java index 19efe20620..3220350558 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java @@ -116,8 +116,7 @@ public class ShellExecutor extends AbstractShell { * @return the output of the executed command. * @throws IOException errors */ - public static String execCommand(Map env, String... cmd) - throws IOException { + public static String execCommand(Map env, String... cmd) throws IOException { return execCommand(env, cmd, 0L); } @@ -154,7 +153,6 @@ public class ShellExecutor extends AbstractShell { return (output == null) ? "" : output.toString(); } - /** * Returns the commands of this instance. * Arguments with spaces in are presented with quotes round; other diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java index 51bf1957e4..f7719b4728 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.spi.common.UiChannelFactory; -import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; +import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; public interface TaskChannelFactory extends UiChannelFactory, PrioritySPI { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index 11e9b97e82..648f17cdda 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -18,8 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api; import java.time.Duration; -import java.util.Arrays; -import java.util.List; public class TaskConstants { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java index 22365f9f81..16e0a0aa65 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java @@ -33,5 +33,5 @@ public enum DataType { * 8 Boolean * 9 list */ - VARCHAR,INTEGER,LONG,FLOAT,DOUBLE,DATE,TIME,TIMESTAMP,BOOLEAN,LIST + VARCHAR, INTEGER, LONG, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, BOOLEAN, LIST } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java index 0b24dad861..539f2e5712 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java @@ -22,7 +22,6 @@ package org.apache.dolphinscheduler.plugin.task.api.enums; */ public enum DependResult { - /** * 0 success * 1 waiting diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java index 75a02e7766..2a60a5e8f3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java @@ -22,5 +22,5 @@ package org.apache.dolphinscheduler.plugin.task.api.enums; */ public enum DependentRelation { - AND,OR; + AND, OR; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java index df76ed1fd8..7bb4112ecd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java @@ -24,5 +24,5 @@ public enum Direct { /** * 0 in; 1 out; */ - IN,OUT + IN, OUT } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java index 9a2818436e..3d6c27a1aa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue; * task timeout strategy */ public enum TaskTimeoutStrategy { + /** * 0 warn * 1 failed diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java index b9298cec92..b65dc07152 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java @@ -21,6 +21,7 @@ package org.apache.dolphinscheduler.plugin.task.api.enums; * UDF type */ public enum UdfType { + /** * 0 hive; 1 spark */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java index 964ef7f8e5..c1fec2e857 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java @@ -26,16 +26,17 @@ import com.fasterxml.jackson.annotation.JsonValue; * quality task result check type */ public enum CheckType { + /** * 0-comparison_minus_statistics * 1-statistics_minus_comparison * 2-statistics_comparison_percentage * 3-statistics_comparison_difference_comparison_percentage */ - COMPARISON_MINUS_STATISTICS(0,"comparison_minus_statistics"), - STATISTICS_MINUS_COMPARISON(1,"statistics_minus_comparison"), - STATISTICS_COMPARISON_PERCENTAGE(2,"statistics_comparison_percentage"), - STATISTICS_COMPARISON_DIFFERENCE_COMPARISON_PERCENTAGE(3,"statistics_comparison_difference_comparison_percentage"); + COMPARISON_MINUS_STATISTICS(0, "comparison_minus_statistics"), + STATISTICS_MINUS_COMPARISON(1, "statistics_minus_comparison"), + STATISTICS_COMPARISON_PERCENTAGE(2, "statistics_comparison_percentage"), + STATISTICS_COMPARISON_DIFFERENCE_COMPARISON_PERCENTAGE(3, "statistics_comparison_difference_comparison_percentage"); CheckType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public enum CheckType { static { for (CheckType type : CheckType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public enum CheckType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java index f3d901a1b4..0e81d64f84 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java @@ -26,12 +26,13 @@ import com.fasterxml.jackson.annotation.JsonValue; * connector type */ public enum ConnectorType { + /** * 0-jdbc * 1-hive */ - JDBC(0,"JDBC"), - HIVE(1,"HIVE"); + JDBC(0, "JDBC"), + HIVE(1, "HIVE"); ConnectorType(int code, String description) { this.code = code; @@ -54,7 +55,7 @@ public enum ConnectorType { static { for (ConnectorType type : ConnectorType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -64,4 +65,4 @@ public enum ConnectorType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java index 34bc6a65a0..1c139a31a9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java @@ -24,6 +24,7 @@ import java.util.Map; * failure policy when dqs task node failed. */ public enum DqFailureStrategy { + /** * 0-alert and continue when dqc tasks failed. * 1-alert and block when dqc tasks failed. @@ -51,7 +52,7 @@ public enum DqFailureStrategy { static { for (DqFailureStrategy type : DqFailureStrategy.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java index d927d57270..710db2e4ae 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java @@ -26,14 +26,15 @@ import com.fasterxml.jackson.annotation.JsonValue; * data quality task state */ public enum DqTaskState { + /** * 0-default * 1-success * 2-failure */ - DEFAULT(0,"default"), - SUCCESS(1,"success"), - FAILURE(2,"failure"); + DEFAULT(0, "default"), + SUCCESS(1, "success"), + FAILURE(2, "failure"); DqTaskState(int code, String description) { this.code = code; @@ -56,7 +57,7 @@ public enum DqTaskState { static { for (DqTaskState type : DqTaskState.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -66,4 +67,4 @@ public enum DqTaskState { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java index ae6415226f..f47cd12478 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java @@ -26,14 +26,15 @@ import com.fasterxml.jackson.annotation.JsonValue; * execute sql type */ public enum ExecuteSqlType { + /** * 0-middle * 1-statistics * 2-comparison */ - MIDDLE(0,"middle"), - STATISTICS(1,"statistics"), - COMPARISON(2,"comparison"); + MIDDLE(0, "middle"), + STATISTICS(1, "statistics"), + COMPARISON(2, "comparison"); ExecuteSqlType(int code, String description) { this.code = code; @@ -56,7 +57,7 @@ public enum ExecuteSqlType { static { for (ExecuteSqlType type : ExecuteSqlType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -66,4 +67,4 @@ public enum ExecuteSqlType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java index 5489fb1cf8..938b884951 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java @@ -26,16 +26,17 @@ import com.fasterxml.jackson.annotation.JsonValue; * frontend form input entry type */ public enum InputType { + /** * 0-default * 1-statistics * 2-comparison * 3-check */ - DEFAULT(0,"default"), - STATISTICS(1,"statistics"), - COMPARISON(2,"comparison"), - CHECK(3,"check"); + DEFAULT(0, "default"), + STATISTICS(1, "statistics"), + COMPARISON(2, "comparison"), + CHECK(3, "check"); InputType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public enum InputType { static { for (InputType type : InputType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public enum InputType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java index 898dc12d76..08e55f6e9d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonValue; * operator type */ public enum OperatorType { + /** * 0-equal * 1-little than @@ -34,12 +35,12 @@ public enum OperatorType { * 4-great and equal * 5-not equal */ - EQ(0,"equal"), - LT(1,"little than"), - LE(2,"little and equal"), - GT(3,"great than"), - GE(4,"great and equal"), - NE(5,"not equal"); + EQ(0, "equal"), + LT(1, "little than"), + LE(2, "little and equal"), + GT(3, "great than"), + GE(4, "great and equal"), + NE(5, "not equal"); OperatorType(int code, String description) { this.code = code; @@ -62,7 +63,7 @@ public enum OperatorType { static { for (OperatorType type : OperatorType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -72,4 +73,4 @@ public enum OperatorType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java index 9e99c96aa7..20da3894e3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java @@ -26,16 +26,17 @@ import com.fasterxml.jackson.annotation.JsonValue; * form options source type */ public enum OptionSourceType { + /** * 0-default * 1-datasource_id * 2-datasource_type * 3-comparison_type */ - DEFAULT(0,"default"), - DATASOURCE_ID(1,"datasource_id"), - DATASOURCE_TYPE(2,"datasource_type"), - COMPARISON_TYPE(3,"comparison_type"); + DEFAULT(0, "default"), + DATASOURCE_ID(1, "datasource_id"), + DATASOURCE_TYPE(2, "datasource_type"), + COMPARISON_TYPE(3, "comparison_type"); OptionSourceType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public enum OptionSourceType { static { for (OptionSourceType type : OptionSourceType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public enum OptionSourceType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java index bcede62e4b..46b42621f0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java @@ -26,16 +26,17 @@ import com.fasterxml.jackson.annotation.JsonValue; * rule type */ public enum RuleType { + /** * 0-single_table * 1-single_table_custom_sql * 2-multi_table_accuracy * 3-multi_table_comparison */ - SINGLE_TABLE(0,"single_table"), - SINGLE_TABLE_CUSTOM_SQL(1,"single_table_custom_sql"), - MULTI_TABLE_ACCURACY(2,"multi_table_accuracy"), - MULTI_TABLE_COMPARISON(3,"multi_table_comparison"); + SINGLE_TABLE(0, "single_table"), + SINGLE_TABLE_CUSTOM_SQL(1, "single_table_custom_sql"), + MULTI_TABLE_ACCURACY(2, "multi_table_accuracy"), + MULTI_TABLE_COMPARISON(3, "multi_table_comparison"); RuleType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public enum RuleType { static { for (RuleType type : RuleType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public enum RuleType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java index d63bf14cb7..0e1150af03 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java @@ -26,16 +26,17 @@ import com.fasterxml.jackson.annotation.JsonValue; * rule input entry value type */ public enum ValueType { + /** * 0-string * 1-list * 2-number * 3-sql */ - STRING(0,"string"), - LIST(1,"list"), - NUMBER(2,"number"), - LIKE_SQL(3,"sql"); + STRING(0, "string"), + LIST(1, "list"), + NUMBER(2, "number"), + LIKE_SQL(3, "sql"); ValueType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public enum ValueType { static { for (ValueType type : ValueType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public enum ValueType { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java index 4567076405..9cab4ed419 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.k8s; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; -import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.k8s.impl.K8sTaskExecutor; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; public abstract class AbstractK8sTask extends AbstractRemoteTask { + /** * process task */ @@ -37,7 +37,7 @@ public abstract class AbstractK8sTask extends AbstractRemoteTask { */ protected AbstractK8sTask(TaskExecutionContext taskRequest) { super(taskRequest); - this.abstractK8sTaskExecutor = new K8sTaskExecutor(logger,taskRequest); + this.abstractK8sTaskExecutor = new K8sTaskExecutor(logger, taskRequest); } // todo split handle to submit and track @@ -49,7 +49,7 @@ public abstract class AbstractK8sTask extends AbstractRemoteTask { setAppIds(response.getAppIds()); } catch (Exception e) { exitStatusCode = -1; - throw new TaskException("k8s process failure",e); + throw new TaskException("k8s process failure", e); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java index 1d619b26b2..d38064a07d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.K8sUtils; import org.slf4j.Logger; public abstract class AbstractK8sTaskExecutor { + protected Logger logger; protected TaskExecutionContext taskRequest; protected K8sUtils k8sUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java index 86caf43934..b0940ae83f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java @@ -82,12 +82,12 @@ public class K8sTaskMainParameters { @Override public String toString() { return "K8sTaskMainParameters{" - + "image='" + image + '\'' - + ", namespaceName='" + namespaceName + '\'' - + ", clusterName='" + clusterName + '\'' - + ", minCpuCores=" + minCpuCores - + ", minMemorySpace=" + minMemorySpace - + ", paramsMap=" + paramsMap - + '}'; + + "image='" + image + '\'' + + ", namespaceName='" + namespaceName + '\'' + + ", clusterName='" + clusterName + '\'' + + ", minCpuCores=" + minCpuCores + + ", minMemorySpace=" + minMemorySpace + + ", paramsMap=" + paramsMap + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java index 4cf027c00c..d6d133f63a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java @@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.plugin.task.api.loop; import lombok.NonNull; -public interface LoopTaskDefinition { +public interface LoopTaskDefinition { + /** * The task name of this loop task, e.g: K8sPodTask */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java index 6d66dfd1cb..78c610a87b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java @@ -21,6 +21,7 @@ package org.apache.dolphinscheduler.plugin.task.api.loop; * The return result of {@link LoopTaskQueryStatusMethodDefinition#queryTaskInstanceStatus(LoopTaskInstanceInfo)}. */ public interface LoopTaskInstanceStatus { + /** * Judge if the task instance is finished. * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java index 0c0b5067a6..344050eab3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java @@ -20,7 +20,8 @@ package org.apache.dolphinscheduler.plugin.task.api.loop; import lombok.NonNull; public interface LoopTaskMethodDefinition { + @NonNull LoopTaskMethodType getLoopTaskMethodType(); - + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java index 2e4582fb1b..47756ce43a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java @@ -24,11 +24,13 @@ import lombok.Data; @Data public class LoopTaskYamlDefinition implements Serializable { + // todo: support multiple services private LoopTaskServiceYamlDefinition service; @Data public static class LoopTaskServiceYamlDefinition implements Serializable { + private String name; private String type; private LoopTaskAPIYamlDefinition api; @@ -36,6 +38,7 @@ public class LoopTaskYamlDefinition implements Serializable { @Data public static class LoopTaskAPIYamlDefinition implements Serializable { + private LoopTaskSubmitMethodYamlDefinition submit; private LoopTaskQueryStateYamlDefinition queryState; private LoopTaskCancelYamlDefinition cancel; @@ -44,6 +47,7 @@ public class LoopTaskYamlDefinition implements Serializable { @Data @SuppressWarnings("checkstyle:ModifierOrder") public static abstract class LoopTaskMethodYamlDefinition { + private String url; private String method; private String dataType; @@ -68,6 +72,7 @@ public class LoopTaskYamlDefinition implements Serializable { @Data public static class LoopTaskQueryStateYamlDefinition extends LoopTaskMethodYamlDefinition { + /** * Used to extract taskInstance finished state from response * todo: we need to support the function to calculate the finished state diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java index 9f09a6c476..524362cc3c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java @@ -21,8 +21,7 @@ import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.HttpLoopTa import lombok.NonNull; -public interface TemplateMethodTransformer { +public interface TemplateMethodTransformer { /** * Transform the {@link LoopTaskYamlDefinition.LoopTaskMethodYamlDefinition} to {@link HttpLoopTaskMethodDefinition}. diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java index 2d80d5caa1..0f9b45f245 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java @@ -24,8 +24,9 @@ import org.apache.dolphinscheduler.plugin.task.api.loop.template.http.method.Htt import lombok.NonNull; -public class HttpLoopTaskDefinition implements - LoopTaskDefinition { +public class HttpLoopTaskDefinition + implements + LoopTaskDefinition { private final String taskName; private final HttpLoopTaskSubmitTaskMethodDefinition submitTaskMethod; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java index d5270863cb..88c7de8f74 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java @@ -29,6 +29,7 @@ import lombok.NonNull; @Data @AllArgsConstructor public abstract class HttpLoopTaskMethodDefinition implements LoopTaskMethodDefinition { + protected final String url; protected final String httpMethodType; protected final String dataType; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java index 240cc74bfb..11620164c9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java @@ -29,7 +29,8 @@ import java.util.Map; import javax.annotation.Nullable; public class HttpLoopTaskCancelTaskMethodDefinition extends HttpLoopTaskMethodDefinition - implements LoopTaskCancelMethodDefinition { + implements + LoopTaskCancelMethodDefinition { private final String taskInstanceIdHolder = "${taskInstanceId}"; @@ -69,7 +70,7 @@ public class HttpLoopTaskCancelTaskMethodDefinition extends HttpLoopTaskMethodDe OkHttpUtils.post(url, httpHeaders, requestParams, requestBody); } else { throw new IllegalArgumentException(String.format("http method type: %s is not supported", - httpMethodType)); + httpMethodType)); } } catch (IllegalArgumentException ex) { throw ex; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java index dbc44a932d..3c7740fc11 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java @@ -32,7 +32,8 @@ import java.util.Map; import lombok.NonNull; public class HttpLoopTaskQueryStatusMethodDefinition extends HttpLoopTaskMethodDefinition - implements LoopTaskQueryStatusMethodDefinition { + implements + LoopTaskQueryStatusMethodDefinition { private final String taskInstanceFinishedJPath; // inject the taskInstanceId into template @@ -75,7 +76,7 @@ public class HttpLoopTaskQueryStatusMethodDefinition extends HttpLoopTaskMethodD responseBody = OkHttpUtils.post(url, httpHeaders, requestParams, requestBody); } else { throw new IllegalArgumentException(String.format("http method type: %s is not supported", - httpMethodType)); + httpMethodType)); } } catch (IllegalArgumentException ex) { throw ex; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java index 20852a9cb4..ec6e3af1a6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java @@ -32,7 +32,8 @@ import java.util.Optional; import lombok.NonNull; public class HttpLoopTaskSubmitTaskMethodDefinition extends HttpLoopTaskMethodDefinition - implements LoopTaskSubmitTaskMethodDefinition { + implements + LoopTaskSubmitTaskMethodDefinition { private final String taskInstanceIdJPath; @@ -58,7 +59,7 @@ public class HttpLoopTaskSubmitTaskMethodDefinition extends HttpLoopTaskMethodDe responseBody = OkHttpUtils.post(url, httpHeaders, requestParams, requestBody); } else { throw new IllegalArgumentException(String.format("The request method type: %s is not supported.", - httpMethodType)); + httpMethodType)); } } catch (IllegalArgumentException ex) { throw ex; @@ -67,9 +68,9 @@ public class HttpLoopTaskSubmitTaskMethodDefinition extends HttpLoopTaskMethodDe } Optional taskInstanceIdOptional = JsonPathUtils.read(responseBody, taskInstanceIdJPath); String taskInstanceId = taskInstanceIdOptional.orElseThrow(() -> new RuntimeException(String.format( - "Resolve the taskInstanceId error, responseBody: %s, taskInstanceIdJPath: %s", - responseBody, - taskInstanceIdJPath))); + "Resolve the taskInstanceId error, responseBody: %s, taskInstanceIdJPath: %s", + responseBody, + taskInstanceIdJPath))); return new HttpLoopTaskInstanceInfo(taskInstanceId); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java index bcc95ba5c7..b367b4c441 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java @@ -25,8 +25,9 @@ import java.util.Map; import lombok.NonNull; -public class CancelTemplateMethodTransformer implements - TemplateMethodTransformer { +public class CancelTemplateMethodTransformer + implements + TemplateMethodTransformer { @Override public @NonNull HttpLoopTaskCancelTaskMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskCancelYamlDefinition loopTaskAPIYamlDefinition) { @@ -37,10 +38,10 @@ public class CancelTemplateMethodTransformer implements Map requestParams = loopTaskAPIYamlDefinition.getRequestParams(); Map requestBody = loopTaskAPIYamlDefinition.getRequestBody(); return new HttpLoopTaskCancelTaskMethodDefinition(url, - method, - dataType, - httpHeaders, - requestParams, - requestBody); + method, + dataType, + httpHeaders, + requestParams, + requestBody); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java index 1bf723736d..b00942fbd5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java @@ -29,13 +29,13 @@ import org.apache.commons.lang3.StringUtils; import java.io.FileReader; import java.io.IOException; +import lombok.NonNull; + import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; import com.google.common.base.Preconditions; -import lombok.NonNull; - public class HttpTaskDefinitionParser implements TaskDefinitionParser { @Override @@ -50,12 +50,12 @@ public class HttpTaskDefinitionParser implements TaskDefinitionParser { +public class QueryStateTemplateMethodTransformer + implements + TemplateMethodTransformer { @Override public @NonNull HttpLoopTaskQueryStatusMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskQueryStateYamlDefinition loopTaskAPIYamlDefinition) { @@ -38,11 +39,11 @@ public class QueryStateTemplateMethodTransformer implements Map requestBody = loopTaskAPIYamlDefinition.getRequestBody(); String taskInstanceFinishedJPath = loopTaskAPIYamlDefinition.getTaskInstanceFinishedJPath(); return new HttpLoopTaskQueryStatusMethodDefinition(url, - method, - dataType, - httpHeaders, - requestParams, - requestBody, - taskInstanceFinishedJPath); + method, + dataType, + httpHeaders, + requestParams, + requestBody, + taskInstanceFinishedJPath); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java index 05b476a288..2cc7956f85 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java @@ -25,8 +25,9 @@ import java.util.Map; import lombok.NonNull; -public class SubmitTemplateMethodTransformer implements - TemplateMethodTransformer { +public class SubmitTemplateMethodTransformer + implements + TemplateMethodTransformer { @Override public @NonNull HttpLoopTaskSubmitTaskMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskSubmitMethodYamlDefinition loopTaskAPIYamlDefinition) { @@ -38,11 +39,11 @@ public class SubmitTemplateMethodTransformer implements Map requestBody = loopTaskAPIYamlDefinition.getRequestBody(); String taskInstanceIdJPath = loopTaskAPIYamlDefinition.getTaskInstanceIdJPath(); return new HttpLoopTaskSubmitTaskMethodDefinition(url, - method, - dataType, - httpHeaders, - requestParams, - requestBody, - taskInstanceIdJPath); + method, + dataType, + httpHeaders, + requestParams, + requestBody, + taskInstanceIdJPath); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java index 893cee1d3f..ad5f34de12 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java @@ -24,6 +24,7 @@ import java.util.Objects; * date interval class */ public class DateInterval { + private Date startTime; private Date endTime; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java index 0a9cf3f499..e19d1c1a51 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import lombok.Data; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import lombok.Data; + /** * dependent item */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java index 3501db67c8..08ebeb979e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import lombok.Data; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import java.util.List; +import lombok.Data; + @Data public class DependentTaskModel { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java index 7f51c7d9b2..bac4e651df 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import java.io.Serializable; import java.util.Objects; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java index dd28fc7711..65577890cc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java @@ -50,4 +50,4 @@ public class SwitchResultVo { this.nextNode = (ArrayList) nextNode; } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java index 76c8b9c3fc..d8cfb86316 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java @@ -40,6 +40,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; * job params related class */ public abstract class AbstractParameters implements IParameters { + @Override public abstract boolean checkParameters(); @@ -79,7 +80,7 @@ public abstract class AbstractParameters implements IParameters { Map localParametersMaps = new LinkedHashMap<>(); if (localParams != null) { for (Property property : localParams) { - localParametersMaps.put(property.getProp(),property); + localParametersMaps.put(property.getProp(), property); } } return localParametersMaps; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java index a258ebdac4..922c344aae 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java @@ -26,7 +26,7 @@ import java.util.List; public class ConditionsParameters extends AbstractParameters { - //depend node list and state, only need task name + // depend node list and state, only need task name private List dependTaskList; private DependentRelation dependRelation; @@ -80,9 +80,9 @@ public class ConditionsParameters extends AbstractParameters { public String getConditionResult() { return "{" - + "\"successNode\": [\"" + successNode.get(0) - + "\"],\"failedNode\": [\"" + failedNode.get(0) - + "\"]}"; + + "\"successNode\": [\"" + successNode.get(0) + + "\"],\"failedNode\": [\"" + failedNode.get(0) + + "\"]}"; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java index e0e6b40068..2b90f4a1cc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; -import lombok.Data; -import lombok.EqualsAndHashCode; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + @Data @EqualsAndHashCode(callSuper = true) public class DependentParameters extends AbstractParameters { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java index 02603efa46..bcd914c8be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; @@ -23,6 +23,7 @@ import java.util.List; * job params interface */ public interface IParameters { + /** * check parameters is valid * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java index d9ab201c36..8121c1f778 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.ArrayList; @@ -28,6 +27,7 @@ import java.util.List; * k8s task parameters */ public class K8sTaskParameters extends AbstractParameters { + private String image; private String namespace; private double minCpuCores; @@ -67,8 +67,7 @@ public class K8sTaskParameters extends AbstractParameters { @Override public boolean checkParameters() { - return StringUtils.isNotEmpty(image) && StringUtils.isNotEmpty(namespace) - ; + return StringUtils.isNotEmpty(image) && StringUtils.isNotEmpty(namespace); } @Override @@ -79,10 +78,10 @@ public class K8sTaskParameters extends AbstractParameters { @Override public String toString() { return "K8sTaskParameters{" - + "image='" + image + '\'' - + ", namespace='" + namespace + '\'' - + ", minCpuCores=" + minCpuCores - + ", minMemorySpace=" + minMemorySpace - + '}'; + + "image='" + image + '\'' + + ", namespace='" + namespace + '\'' + + ", minCpuCores=" + minCpuCores + + ", minMemorySpace=" + minMemorySpace + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java index 0f6b4107e3..b6eb88a68a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java @@ -36,6 +36,7 @@ public class ParametersNode { } public static class ParametersNodeBuilder { + private String taskType; private String taskParams; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java index 3616bc57f5..3368313935 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java @@ -37,4 +37,4 @@ public class SubProcessParameters extends AbstractParameters { return this.processDefinitionCode != 0; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java index 502ea14701..efdcba62ee 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java @@ -86,4 +86,4 @@ public class SwitchParameters extends AbstractParameters { this.nextNode = (ArrayList) nextNode; } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java index f257c427ce..c6475c9631 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.SparkParameters; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import java.util.ArrayList; @@ -43,7 +43,7 @@ public class DataQualityParameters extends AbstractParameters { /** * rule input entry value map */ - private Map ruleInputParameter; + private Map ruleInputParameter; /** * spark parameters */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java index b719f95b4e..4d3ec17954 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java @@ -28,5 +28,4 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; }) public abstract class AbstractResourceParameters { - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java index ea9cb7ae5a..963eb9a3b4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java @@ -17,9 +17,10 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters.resource; -import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.dolphinscheduler.spi.enums.DbType; +import com.fasterxml.jackson.annotation.JsonProperty; + public class DataSourceParameters extends AbstractResourceParameters { private DbType type; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java index b09d9f8972..1dd4b42529 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java @@ -35,6 +35,7 @@ import java.util.Map; * business time utils */ public class BusinessTimeUtils { + private BusinessTimeUtils() { throw new IllegalStateException("BusinessTimeUtils class"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java index 15a7c14b76..c106f7932d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.task.api.parser; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import java.util.HashMap; import java.util.Iterator; @@ -36,7 +36,7 @@ public class ParamUtils { * @param paramsMap params map * @return Map of converted */ - public static Map convert(Map paramsMap) { + public static Map convert(Map paramsMap) { if (paramsMap == null) { return null; } @@ -63,9 +63,9 @@ public class ParamUtils { while (iter.hasNext()) { Map.Entry en = iter.next(); Property property = new Property(en.getKey(), Direct.IN, DataType.VARCHAR, en.getValue()); - userDefParamsMaps.put(property.getProp(),property); + userDefParamsMaps.put(property.getProp(), property); } } return userDefParamsMaps; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java index 8d3e265d6e..393a51bf0d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java @@ -73,7 +73,7 @@ public class ParameterUtils { parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); } if (parameterMap != null && null != parameterMap.get(PARAMETER_DATETIME)) { - //Get current time, schedule execute time + // Get current time, schedule execute time String cronTimeStr = parameterMap.get(PARAMETER_DATETIME); cronTime = DateUtils.parse(cronTimeStr, PARAMETER_FORMAT_TIME); } else { @@ -98,7 +98,7 @@ public class ParameterUtils { if (StringUtils.isEmpty(parameterString)) { return parameterString; } - //Get current time, schedule execute time + // Get current time, schedule execute time String cronTimeStr = parameterMap.get(PARAMETER_SHECDULE_TIME); Date cronTime = null; @@ -130,7 +130,8 @@ public class ParameterUtils { * @param value value * @throws Exception errors */ - public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception { + public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, + String value) throws Exception { if (dataType.equals(DataType.VARCHAR)) { stmt.setString(index, value); } else if (dataType.equals(DataType.INTEGER)) { @@ -177,9 +178,9 @@ public class ParameterUtils { ret.append(","); } } - for (Object v : valueList ) { + for (Object v : valueList) { Property newProperty = new Property(); - if (v instanceof Integer) { + if (v instanceof Integer) { newProperty.setType(DataType.INTEGER); } else if (v instanceof Long) { newProperty.setType(DataType.LONG); @@ -215,7 +216,7 @@ public class ParameterUtils { */ public static String replaceScheduleTime(String text, Date scheduleTime) { Map paramsMap = new HashMap<>(); - //if getScheduleTime null ,is current date + // if getScheduleTime null ,is current date if (null == scheduleTime) { scheduleTime = new Date(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java index f266ced99b..3820646364 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java @@ -52,15 +52,16 @@ public class PlaceholderUtils { public static String replacePlaceholders(String value, Map paramsMap, boolean ignoreUnresolvablePlaceholders) { - //replacement tool， parameter key will be replaced by value,if can't match , will throw an exception + // replacement tool， parameter key will be replaced by value,if can't match , will throw an exception PropertyPlaceholderHelper strictHelper = getPropertyPlaceholderHelper(false); - //Non-strict replacement tool implementation, when the position to be replaced does not get the corresponding value, the current position is ignored, and the next position is replaced. + // Non-strict replacement tool implementation, when the position to be replaced does not get the corresponding + // value, the current position is ignored, and the next position is replaced. PropertyPlaceholderHelper nonStrictHelper = getPropertyPlaceholderHelper(true); PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ? nonStrictHelper : strictHelper); - //the PlaceholderResolver to use for replacement + // the PlaceholderResolver to use for replacement return helper.replacePlaceholders(value, new PropertyPlaceholderResolver(value, paramsMap)); } @@ -72,7 +73,8 @@ public class PlaceholderUtils { */ public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, + ignoreUnresolvablePlaceholders); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java index a4b08a6f98..98b6e01dc8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java @@ -48,7 +48,6 @@ public class PropertyPlaceholderHelper { wellKnownSimplePrefixes.put(")", "("); } - private final String placeholderPrefix; private final String placeholderSuffix; @@ -88,8 +87,7 @@ public class PropertyPlaceholderHelper { String simplePrefixForSuffix = wellKnownSimplePrefixes.get(this.placeholderSuffix); if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) { this.simplePrefix = simplePrefixForSuffix; - } - else { + } else { this.simplePrefix = this.placeholderPrefix; } this.valueSeparator = valueSeparator; @@ -106,6 +104,7 @@ public class PropertyPlaceholderHelper { public String replacePlaceholders(String value, final Properties properties) { notNull(properties, "'properties' must not be null"); return replacePlaceholders(value, new PlaceholderResolver() { + @Override public String resolvePlaceholder(String placeholderName) { return properties.getProperty(placeholderName); @@ -126,7 +125,8 @@ public class PropertyPlaceholderHelper { } protected String parseStringValue( - String value, PlaceholderResolver placeholderResolver, Set visitedPlaceholders) { + String value, PlaceholderResolver placeholderResolver, + Set visitedPlaceholders) { StringBuilder result = new StringBuilder(value); @@ -164,18 +164,15 @@ public class PropertyPlaceholderHelper { logger.trace("Resolved placeholder '" + placeholder + "'"); } startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length()); - } - else if (this.ignoreUnresolvablePlaceholders) { + } else if (this.ignoreUnresolvablePlaceholders) { // Proceed with unprocessed value. startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length()); - } - else { + } else { throw new IllegalArgumentException("Could not resolve placeholder '" + placeholder + "'" + " in value \"" + value + "\""); } visitedPlaceholders.remove(originalPlaceholder); - } - else { + } else { startIndex = -1; } } @@ -191,16 +188,13 @@ public class PropertyPlaceholderHelper { if (withinNestedPlaceholder > 0) { withinNestedPlaceholder--; index = index + this.placeholderSuffix.length(); - } - else { + } else { return index; } - } - else if (substringMatch(buf, index, this.simplePrefix)) { + } else if (substringMatch(buf, index, this.simplePrefix)) { withinNestedPlaceholder++; index = index + this.simplePrefix.length(); - } - else { + } else { index++; } } @@ -252,4 +246,3 @@ public class PropertyPlaceholderHelper { } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java index 4080d0d9c7..518b5e5229 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java @@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory; * time place holder utils */ public class TimePlaceholderUtils { + private static final Logger logger = LoggerFactory.getLogger(TimePlaceholderUtils.class); /** @@ -105,7 +106,8 @@ public class TimePlaceholderUtils { * be ignored ({@code true}) or cause an exception ({@code false}) */ private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, + ignoreUnresolvablePlaceholders); } /** @@ -139,7 +141,8 @@ public class TimePlaceholderUtils { arr[i] = N; } else { char c = arr[i - 1]; - if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { + if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR + || c == LEFT_BRACE_CHAR) { arr[i] = N; } } @@ -148,7 +151,8 @@ public class TimePlaceholderUtils { arr[i] = P; } else { char c = arr[i - 1]; - if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { + if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR + || c == LEFT_BRACE_CHAR) { arr[i] = P; } } @@ -285,9 +289,11 @@ public class TimePlaceholderUtils { * @return true or false */ private static boolean compare(String peek, String cur) { - if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) + || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { return true; - } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) + || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { return true; } else if (ADD_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { return true; @@ -300,8 +306,9 @@ public class TimePlaceholderUtils { /** * Placeholder replacement resolver */ - private static class TimePlaceholderResolver implements - PropertyPlaceholderHelper.PlaceholderResolver { + private static class TimePlaceholderResolver + implements + PropertyPlaceholderHelper.PlaceholderResolver { private final String value; @@ -362,7 +369,7 @@ public class TimePlaceholderUtils { value = String.valueOf(timestamp.getTime() / 1000); } else if (expression.startsWith(YEAR_WEEK)) { - value = calculateYearWeek(expression,date); + value = calculateYearWeek(expression, date); } else { Map.Entry entry = calcTimeExpression(expression, date); value = DateUtils.format(entry.getKey(), entry.getValue()); @@ -388,7 +395,7 @@ public class TimePlaceholderUtils { String targetDate = ""; try { - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; @@ -414,10 +421,10 @@ public class TimePlaceholderUtils { */ private static String transformYearWeek(Date date, String format, int weekDay) { Calendar calendar = Calendar.getInstance(); - //Minimum number of days required for the first week of the year + // Minimum number of days required for the first week of the year calendar.setMinimalDaysInFirstWeek(4); - //By default ,Set Monday as the first day of the week + // By default ,Set Monday as the first day of the week switch (weekDay) { case 2: calendar.setFirstDayOfWeek(Calendar.TUESDAY); @@ -449,13 +456,13 @@ public class TimePlaceholderUtils { String weekYearStr = ""; if (weekOfYear < 10 && format.contains(HYPHEN)) { - weekYearStr = String.format("%d%s0%d",year,HYPHEN,weekOfYear); - } else if (weekOfYear >= 10 && format.contains(HYPHEN)){ - weekYearStr = String.format("%d%s%d",year,HYPHEN,weekOfYear); - } else if (weekOfYear < 10){ - weekYearStr = String.format("%d0%d",year,weekOfYear); + weekYearStr = String.format("%d%s0%d", year, HYPHEN, weekOfYear); + } else if (weekOfYear >= 10 && format.contains(HYPHEN)) { + weekYearStr = String.format("%d%s%d", year, HYPHEN, weekOfYear); + } else if (weekOfYear < 10) { + weekYearStr = String.format("%d0%d", year, weekOfYear); } else { - weekYearStr = String.format("%d%d",year,weekOfYear); + weekYearStr = String.format("%d%d", year, weekOfYear); } return weekYearStr; @@ -482,17 +489,17 @@ public class TimePlaceholderUtils { } else if (expression.startsWith(WEEK_END)) { resultEntry = calcWeekEnd(expression, date); } else if (expression.startsWith(MONTH_FIRST_DAY)) { - resultEntry = calcCustomDay(expression,MONTH_FIRST_DAY, date); + resultEntry = calcCustomDay(expression, MONTH_FIRST_DAY, date); } else if (expression.startsWith(MONTH_LAST_DAY)) { - resultEntry = calcCustomDay(expression,MONTH_LAST_DAY, date); + resultEntry = calcCustomDay(expression, MONTH_LAST_DAY, date); } else if (expression.startsWith(THIS_DAY)) { - resultEntry = calcCustomDay(expression,THIS_DAY, date); + resultEntry = calcCustomDay(expression, THIS_DAY, date); } else if (expression.startsWith(LAST_DAY)) { - resultEntry = calcCustomDay(expression,LAST_DAY, date); + resultEntry = calcCustomDay(expression, LAST_DAY, date); } else if (expression.startsWith(WEEK_FIRST_DAY)) { - resultEntry = calcCustomDay(expression,WEEK_FIRST_DAY, date); + resultEntry = calcCustomDay(expression, WEEK_FIRST_DAY, date); } else if (expression.startsWith(WEEK_LAST_DAY)) { - resultEntry = calcCustomDay(expression,WEEK_LAST_DAY, date); + resultEntry = calcCustomDay(expression, WEEK_LAST_DAY, date); } else { resultEntry = calcMinutes(expression, date); } @@ -555,7 +562,7 @@ public class TimePlaceholderUtils { * @param date date * @return calculate time expression with date,format */ - public static Map.Entry calcCustomDay(String expression,String keyDate, Date date) { + public static Map.Entry calcCustomDay(String expression, String keyDate, Date date) { String dataFormat = "yyyy-MM-dd"; Date targetDate = new Date(); @@ -563,12 +570,12 @@ public class TimePlaceholderUtils { case MONTH_FIRST_DAY: dataFormat = expression.substring(MONTH_FIRST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; - targetDate = addMonths(DateUtils.getFirstDayOfMonth(date),calculate(param2)); + targetDate = addMonths(DateUtils.getFirstDayOfMonth(date), calculate(param2)); } else { targetDate = DateUtils.getFirstDayOfMonth(date); } @@ -577,12 +584,12 @@ public class TimePlaceholderUtils { case MONTH_LAST_DAY: dataFormat = expression.substring(MONTH_LAST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; - Date lastMonthDay = addMonths(date,calculate(param2)); + Date lastMonthDay = addMonths(date, calculate(param2)); targetDate = DateUtils.getLastDayOfMonth(lastMonthDay); @@ -601,7 +608,7 @@ public class TimePlaceholderUtils { case WEEK_FIRST_DAY: dataFormat = expression.substring(WEEK_FIRST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; @@ -614,7 +621,7 @@ public class TimePlaceholderUtils { case WEEK_LAST_DAY: dataFormat = expression.substring(WEEK_LAST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; @@ -755,7 +762,7 @@ public class TimePlaceholderUtils { } else { calcExpression = String.format("60*24*(%s)%s", minuteExpression.substring(0, index), - minuteExpression.substring(index)); + minuteExpression.substring(index)); } return calculate(calcExpression); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java index 51ab17e156..15fb4d5ee4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java @@ -18,5 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.stream; public interface StreamTask { + public void savePoint() throws Exception; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java index a3bf9e5e23..6c35b3cec4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java @@ -20,9 +20,6 @@ package org.apache.dolphinscheduler.plugin.task.api.stream; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public interface StreamTaskChannel extends TaskChannel { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java index abecf94a34..88fc080d44 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java @@ -179,7 +179,7 @@ public class DependentDateUtils { */ public static List getLastWeekOneDayInterval(Date businessDate, int dayOfWeek) { Date mondayThisWeek = DateUtils.getMonday(businessDate); - Date sunday = DateUtils.getSomeDay(mondayThisWeek, - 1); + Date sunday = DateUtils.getSomeDay(mondayThisWeek, -1); Date monday = DateUtils.getMonday(sunday); Date destDay = DateUtils.getSomeDay(monday, dayOfWeek - 1); return getDateIntervalListBetweenTwoDates(destDay, destDay); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java index e6e8cb219d..452ed3af86 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java @@ -20,14 +20,14 @@ package org.apache.dolphinscheduler.plugin.task.api.utils; import java.util.Optional; import java.util.Set; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.internal.DefaultsImpl; - import lombok.NonNull; import lombok.experimental.UtilityClass; import net.minidev.json.JSONArray; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.internal.DefaultsImpl; + @UtilityClass public class JsonPathUtils { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java index 84cfe2a730..db6625d0c6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java @@ -58,7 +58,7 @@ public class OkHttpUtils { Request.Builder requestBuilder = new Request.Builder().url(finalUrl); if (requestBodyMap != null) { requestBuilder = requestBuilder.post(RequestBody.create(MediaType.parse("application/json"), - JSONUtils.toJsonString(requestBodyMap))); + JSONUtils.toJsonString(requestBodyMap))); } try (Response response = CLIENT.newCall(requestBuilder.build()).execute()) { return getResponseBody(response); @@ -91,8 +91,8 @@ public class OkHttpUtils { private static String getResponseBody(@NonNull Response response) throws IOException { if (response.code() != HttpStatus.SC_OK || response.body() == null) { throw new RuntimeException(String.format("Request execute failed, httpCode: %s, httpBody: %s", - response.code(), - response.body())); + response.code(), + response.body())); } return response.body().string(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java index 2dcf3efce4..1246d85f15 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java @@ -62,6 +62,7 @@ public class RetryUtils { @Data public static final class RetryPolicy { + /** * The max retry times */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java index b52a9a099b..c17902f568 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java @@ -156,7 +156,8 @@ public class JSONUtilsTest { Assertions.assertEquals("", JSONUtils.getNodeString("{\"bar\":\"foo\"}", "key")); Assertions.assertEquals("foo", JSONUtils.getNodeString("{\"bar\":\"foo\"}", "bar")); Assertions.assertEquals("[1,2,3]", JSONUtils.getNodeString("{\"bar\": [1,2,3]}", "bar")); - Assertions.assertEquals("{\"1\":\"2\",\"2\":3}", JSONUtils.getNodeString("{\"bar\": {\"1\":\"2\",\"2\":3}}", "bar")); + Assertions.assertEquals("{\"1\":\"2\",\"2\":3}", + JSONUtils.getNodeString("{\"bar\": {\"1\":\"2\",\"2\":3}}", "bar")); } @Test @@ -216,7 +217,8 @@ public class JSONUtilsTest { Assertions.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJsonString(map)); Assertions.assertEquals(String.valueOf((Object) null), JSONUtils.toJsonString(null)); - Assertions.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJsonString(map, SerializationFeature.WRITE_NULL_MAP_VALUES)); + Assertions.assertEquals("{\"foo\":\"bar\"}", + JSONUtils.toJsonString(map, SerializationFeature.WRITE_NULL_MAP_VALUES)); } @Test diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 65d8be407d..ea65116a5b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -60,7 +60,8 @@ public class ParameterUtilsTest { // replace variable ${} form parameterMap.put("testParameter2", "${testParameter}"); - Assertions.assertEquals(parameterString, PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true)); + Assertions.assertEquals(parameterString, + PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true)); // replace time $[...] form, eg. $[yyyyMMdd] Date cronTime = new Date(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml index 6ed2077155..f57d1c0780 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-blocking jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java index a8f0660cb6..3398e1e405 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java @@ -29,6 +29,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class BlockingTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new BlockingTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml index 6929353a9b..ee6db0c85f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml @@ -15,13 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-chunjun jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java index 4d6cb06ab1..6980ca549a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java @@ -113,8 +113,8 @@ public class ChunJunParameters extends AbstractParameters { public boolean checkParameters() { if (customConfig == Flag.NO.ordinal()) { return dataSource != 0 && dataTarget != 0 - && StringUtils.isNotEmpty(sql) - && StringUtils.isNotEmpty(targetTable); + && StringUtils.isNotEmpty(sql) + && StringUtils.isNotEmpty(targetTable); } else { return StringUtils.isNotEmpty(json); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java index a690dcd383..f152417aa9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.chunjun; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; @@ -34,6 +34,9 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.Flag; import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.SystemUtils; + import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -48,13 +51,11 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; - /** * chunjun task */ public class ChunJunTask extends AbstractTask { + /** * chunjun path */ @@ -85,7 +86,7 @@ public class ChunJunTask extends AbstractTask { this.taskExecutionContext = taskExecutionContext; this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, - taskExecutionContext, logger); + taskExecutionContext, logger); } /** @@ -139,12 +140,11 @@ public class ChunJunTask extends AbstractTask { * @return * @throws Exception */ - private String buildChunJunJsonFile(Map paramsMap) - throws Exception { + private String buildChunJunJsonFile(Map paramsMap) throws Exception { // generate json String fileName = String.format("%s/%s_job.json", - taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId()); + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); String json = null; @@ -173,13 +173,12 @@ public class ChunJunTask extends AbstractTask { * @return shell command file name * @throws Exception if error throws Exception */ - private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) - throws Exception { + private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) throws Exception { // generate scripts String fileName = String.format("%s/%s_node.%s", - taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId(), - SystemUtils.IS_OS_WINDOWS ? "bat" : "sh"); + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId(), + SystemUtils.IS_OS_WINDOWS ? "bat" : "sh"); Path path = new File(fileName).toPath(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml index 97e6c09815..d4266ae9f9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-conditions jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java index e0a41c1d0d..16b083d826 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java @@ -34,6 +34,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class ConditionsTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new ConditionsTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml index 4311fa80c8..cc64b6cdcb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dataquality jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java index 3270605842..e6ceae8640 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java @@ -21,8 +21,8 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.JSONUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java index 8f05e9cbc6..8bc820f34c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java @@ -38,7 +38,8 @@ public class DataQualityException extends RuntimeException { super(cause); } - protected DataQualityException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + protected DataQualityException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java index b7a550a9de..1fb019d879 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java @@ -67,15 +67,15 @@ public class RuleManager { + "${update_time} as update_time "; public static final String DEFAULT_COMPARISON_WRITER_SQL = - BASE_SQL + "from ${statistics_table} full join ${comparison_table}"; + BASE_SQL + "from ${statistics_table} full join ${comparison_table}"; public static final String MULTI_TABLE_COMPARISON_WRITER_SQL = - BASE_SQL + BASE_SQL + "from ( ${statistics_execute_sql} ) tmp1 " + "join ( ${comparison_execute_sql} ) tmp2"; public static final String SINGLE_TABLE_CUSTOM_SQL_WRITER_SQL = - BASE_SQL + BASE_SQL + "from ( ${statistics_table} ) tmp1 " + "join ${comparison_table}"; public static final String TASK_STATISTICS_VALUE_WRITER_SQL = @@ -91,7 +91,8 @@ public class RuleManager { + "${update_time} as update_time " + "from ${statistics_table}"; - public RuleManager(Map inputParameterValue, DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { + public RuleManager(Map inputParameterValue, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { this.inputParameterValue = inputParameterValue; this.dataQualityTaskExecutionContext = dataQualityTaskExecutionContext; } @@ -104,7 +105,8 @@ public class RuleManager { Map inputParameterValueResult = RuleParserUtils.getInputParameterMapFromEntryList( - JSONUtils.toList(dataQualityTaskExecutionContext.getRuleInputEntryList(), DqRuleInputEntry.class)); + JSONUtils.toList(dataQualityTaskExecutionContext.getRuleInputEntryList(), + DqRuleInputEntry.class)); inputParameterValueResult.putAll(inputParameterValue); inputParameterValueResult.putAll(BusinessTimeUtils.getBusinessTime(CommandType.START_PROCESS, new Date())); inputParameterValueResult.putIfAbsent(COMPARISON_TYPE, NONE_COMPARISON_TYPE); @@ -131,4 +133,4 @@ public class RuleManager { return ruleParser.parse(inputParameterValueResult, dataQualityTaskExecutionContext); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java index a775bde6d4..cbcf908464 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java @@ -30,12 +30,12 @@ public class BaseConfig { private String type; @JsonProperty("config") - private Map config; + private Map config; public BaseConfig() { } - public BaseConfig(String type, Map config) { + public BaseConfig(String type, Map config) { this.type = type; this.config = config; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java index eff3132978..3773932bac 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java @@ -44,7 +44,8 @@ public class DataQualityConfiguration { @JsonProperty("writers") private List writerConfigs; - public DataQualityConfiguration() {} + public DataQualityConfiguration() { + } public DataQualityConfiguration(String name, List readerConfigs, diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java index b4c49b89d6..5da079e69c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java @@ -27,8 +27,8 @@ public class EnvConfig extends BaseConfig { public EnvConfig() { } - public EnvConfig(String type, Map config) { - super(type,config); + public EnvConfig(String type, Map config) { + super(type, config); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java index de7b6e6230..eb3801efc7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java @@ -28,6 +28,6 @@ import java.util.Map; */ public interface IRuleParser { - DataQualityConfiguration parse(Map inputParameterValue, + DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java index 666a16b433..05e09efe27 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java @@ -26,7 +26,8 @@ public class MappingColumn { private String operator; private String targetField; - public MappingColumn() {} + public MappingColumn() { + } public MappingColumn(String srcField, String operator, String targetField) { this.srcField = srcField; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java index 314a6272da..88c761690a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java @@ -45,28 +45,29 @@ public class MultiTableAccuracyRuleParser implements IRuleParser { public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List dqRuleExecuteSqlList = - JSONUtils.toList(context.getExecuteSqlList(),DqRuleExecuteSql.class); + JSONUtils.toList(context.getExecuteSqlList(), DqRuleExecuteSql.class); DqRuleExecuteSql statisticsSql = RuleParserUtils.getExecuteSqlListByType( dqRuleExecuteSqlList, ExecuteSqlType.STATISTICS).get(0); - inputParameterValue.put(STATISTICS_TABLE,statisticsSql.getTableAlias()); + inputParameterValue.put(STATISTICS_TABLE, statisticsSql.getTableAlias()); int index = 1; List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = new ArrayList<>(); - List mappingColumnList = RuleParserUtils.getMappingColumnList(inputParameterValue.get(MAPPING_COLUMNS)); + List mappingColumnList = + RuleParserUtils.getMappingColumnList(inputParameterValue.get(MAPPING_COLUMNS)); - //get on clause - inputParameterValue.put(ON_CLAUSE, RuleParserUtils.getOnClause(mappingColumnList,inputParameterValue)); - //get where clause - inputParameterValue.put(WHERE_CLAUSE, RuleParserUtils.getWhereClause(mappingColumnList,inputParameterValue)); + // get on clause + inputParameterValue.put(ON_CLAUSE, RuleParserUtils.getOnClause(mappingColumnList, inputParameterValue)); + // get where clause + inputParameterValue.put(WHERE_CLAUSE, RuleParserUtils.getWhereClause(mappingColumnList, inputParameterValue)); index = RuleParserUtils.replaceExecuteSqlPlaceholder( dqRuleExecuteSqlList, @@ -76,11 +77,11 @@ public class MultiTableAccuracyRuleParser implements IRuleParser { String writerSql = RuleManager.DEFAULT_COMPARISON_WRITER_SQL; if (context.isCompareWithFixedValue()) { - writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}",""); + writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}", ""); } List writerConfigList = RuleParserUtils.getAllWriterConfigList(inputParameterValue, - context, index, transformerConfigList, writerSql,RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); + context, index, transformerConfigList, writerSql, RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); return new DataQualityConfiguration( context.getRuleName(), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java index bc04fd629e..b6a130366b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java @@ -39,13 +39,14 @@ public class MultiTableComparisonRuleParser implements IRuleParser { DataQualityTaskExecutionContext context) throws DataQualityException { List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = new ArrayList<>(); List writerConfigList = RuleParserUtils.getWriterConfigList( - ParameterUtils.convertParameterPlaceholders(RuleManager.MULTI_TABLE_COMPARISON_WRITER_SQL,inputParameterValue), + ParameterUtils.convertParameterPlaceholders(RuleManager.MULTI_TABLE_COMPARISON_WRITER_SQL, + inputParameterValue), context); return new DataQualityConfiguration( diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java index 09d9e63ac7..d4c8adc4e4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java @@ -38,31 +38,31 @@ public class SingleTableCustomSqlRuleParser implements IRuleParser { public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List dqRuleExecuteSqlList = - JSONUtils.toList(context.getExecuteSqlList(),DqRuleExecuteSql.class); + JSONUtils.toList(context.getExecuteSqlList(), DqRuleExecuteSql.class); int index = 1; List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = RuleParserUtils - .getSingleTableCustomSqlTransformerConfigList(index,inputParameterValue); + .getSingleTableCustomSqlTransformerConfigList(index, inputParameterValue); - //replace the placeholder in execute sql list + // replace the placeholder in execute sql list index = RuleParserUtils.replaceExecuteSqlPlaceholder( - dqRuleExecuteSqlList, - index, - inputParameterValue, - transformerConfigList); + dqRuleExecuteSqlList, + index, + inputParameterValue, + transformerConfigList); String writerSql = RuleManager.SINGLE_TABLE_CUSTOM_SQL_WRITER_SQL; if (context.isCompareWithFixedValue()) { - writerSql = writerSql.replaceAll("join \\$\\{comparison_table}",""); + writerSql = writerSql.replaceAll("join \\$\\{comparison_table}", ""); } List writerConfigList = RuleParserUtils.getAllWriterConfigList(inputParameterValue, - context, index, transformerConfigList, writerSql,RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); + context, index, transformerConfigList, writerSql, RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); return new DataQualityConfiguration( context.getRuleName(), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java index 92ead2a007..456d9c7a37 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java @@ -42,35 +42,35 @@ public class SingleTableRuleParser implements IRuleParser { public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List dqRuleExecuteSqlList = - JSONUtils.toList(context.getExecuteSqlList(),DqRuleExecuteSql.class); + JSONUtils.toList(context.getExecuteSqlList(), DqRuleExecuteSql.class); DqRuleExecuteSql statisticsSql = RuleParserUtils.getExecuteSqlListByType(dqRuleExecuteSqlList, ExecuteSqlType.STATISTICS).get(0); - inputParameterValue.put(STATISTICS_TABLE,statisticsSql.getTableAlias()); + inputParameterValue.put(STATISTICS_TABLE, statisticsSql.getTableAlias()); int index = 1; List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = new ArrayList<>(); - //replace the placeholder in execute sql list + // replace the placeholder in execute sql list index = RuleParserUtils.replaceExecuteSqlPlaceholder( - dqRuleExecuteSqlList, - index, - inputParameterValue, - transformerConfigList); + dqRuleExecuteSqlList, + index, + inputParameterValue, + transformerConfigList); String writerSql = RuleManager.DEFAULT_COMPARISON_WRITER_SQL; if (context.isCompareWithFixedValue()) { - writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}",""); + writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}", ""); } List writerConfigList = RuleParserUtils.getAllWriterConfigList(inputParameterValue, - context, index, transformerConfigList, writerSql,RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); + context, index, transformerConfigList, writerSql, RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); return new DataQualityConfiguration( context.getRuleName(), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java index ead7187081..3c1d78a0d7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java @@ -100,12 +100,12 @@ public class RuleParserUtils { private static final String WHERE_TARGET_FILTER = "WHERE (${target_filter})"; public static List getReaderConfigList( - Map inputParameterValue, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + Map inputParameterValue, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List readerConfigList = new ArrayList<>(); - //all the rule need the source config + // all the rule need the source config if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getSourceConnectorType())) { BaseConnectionParam sourceDataSource = (BaseConnectionParam) DataSourceUtils.buildConnectionParams( @@ -113,17 +113,19 @@ public class RuleParserUtils { dataQualityTaskExecutionContext.getSourceConnectionParams()); BaseConfig sourceBaseConfig = new BaseConfig(); sourceBaseConfig.setType(dataQualityTaskExecutionContext.getSourceConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (sourceDataSource != null) { - config.put(DATABASE,sourceDataSource.getDatabase()); - config.put(TABLE,inputParameterValue.get(SRC_TABLE)); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getSourceType()),sourceDataSource)); - config.put(USER,sourceDataSource.getUser()); - config.put(PASSWORD,sourceDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getSourceType()))); + config.put(DATABASE, sourceDataSource.getDatabase()); + config.put(TABLE, inputParameterValue.get(SRC_TABLE)); + config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getSourceType()), + sourceDataSource)); + config.put(USER, sourceDataSource.getUser()); + config.put(PASSWORD, sourceDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getSourceType()))); String outputTable = sourceDataSource.getDatabase() + "_" + inputParameterValue.get(SRC_TABLE); - config.put(OUTPUT_TABLE,outputTable); - inputParameterValue.put(SRC_TABLE,outputTable); + config.put(OUTPUT_TABLE, outputTable); + inputParameterValue.put(SRC_TABLE, outputTable); } sourceBaseConfig.setConfig(config); @@ -138,17 +140,19 @@ public class RuleParserUtils { dataQualityTaskExecutionContext.getTargetConnectionParams()); BaseConfig targetBaseConfig = new BaseConfig(); targetBaseConfig.setType(dataQualityTaskExecutionContext.getTargetConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (targetDataSource != null) { - config.put(DATABASE,targetDataSource.getDatabase()); - config.put(TABLE,inputParameterValue.get(TARGET_TABLE)); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()),targetDataSource)); - config.put(USER,targetDataSource.getUser()); - config.put(PASSWORD,targetDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType()))); + config.put(DATABASE, targetDataSource.getDatabase()); + config.put(TABLE, inputParameterValue.get(TARGET_TABLE)); + config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()), + targetDataSource)); + config.put(USER, targetDataSource.getUser()); + config.put(PASSWORD, targetDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType()))); String outputTable = targetDataSource.getDatabase() + "_" + inputParameterValue.get(TARGET_TABLE); - config.put(OUTPUT_TABLE,outputTable); - inputParameterValue.put(TARGET_TABLE,outputTable); + config.put(OUTPUT_TABLE, outputTable); + inputParameterValue.put(TARGET_TABLE, outputTable); } targetBaseConfig.setConfig(config); @@ -159,26 +163,29 @@ public class RuleParserUtils { } public static int replaceExecuteSqlPlaceholder(List executeSqlList, - int index, Map inputParameterValueResult, - List transformerConfigList) { - List midExecuteSqlDefinitionList - = getExecuteSqlListByType(executeSqlList, ExecuteSqlType.MIDDLE); - - List statisticsExecuteSqlDefinitionList - = getExecuteSqlListByType(executeSqlList, ExecuteSqlType.STATISTICS); - - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),AND_SRC_FILTER); - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),WHERE_SRC_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),AND_SRC_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),WHERE_SRC_FILTER); - - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),AND_TARGET_FILTER); - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),WHERE_TARGET_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),AND_TARGET_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),WHERE_TARGET_FILTER); + int index, Map inputParameterValueResult, + List transformerConfigList) { + List midExecuteSqlDefinitionList = + getExecuteSqlListByType(executeSqlList, ExecuteSqlType.MIDDLE); + + List statisticsExecuteSqlDefinitionList = + getExecuteSqlListByType(executeSqlList, ExecuteSqlType.STATISTICS); + + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), AND_SRC_FILTER); + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), WHERE_SRC_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), AND_SRC_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), + WHERE_SRC_FILTER); + + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), AND_TARGET_FILTER); + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), WHERE_TARGET_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), + AND_TARGET_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), + WHERE_TARGET_FILTER); if (CollectionUtils.isNotEmpty(midExecuteSqlDefinitionList)) { - for (DqRuleExecuteSql executeSqlDefinition:midExecuteSqlDefinitionList) { + for (DqRuleExecuteSql executeSqlDefinition : midExecuteSqlDefinitionList) { index = setTransformerConfig( index, inputParameterValueResult, @@ -188,7 +195,7 @@ public class RuleParserUtils { } if (CollectionUtils.isNotEmpty(statisticsExecuteSqlDefinitionList)) { - for (DqRuleExecuteSql executeSqlDefinition:statisticsExecuteSqlDefinitionList) { + for (DqRuleExecuteSql executeSqlDefinition : statisticsExecuteSqlDefinitionList) { index = setTransformerConfig( index, inputParameterValueResult, @@ -201,59 +208,61 @@ public class RuleParserUtils { } private static int setTransformerConfig(int index, - Map inputParameterValueResult, - List transformerConfigList, - DqRuleExecuteSql executeSqlDefinition) { - Map config = new HashMap<>(); - config.put(INDEX,index++); - config.put(SQL, ParameterUtils.convertParameterPlaceholders(executeSqlDefinition.getSql(),inputParameterValueResult)); - config.put(OUTPUT_TABLE,executeSqlDefinition.getTableAlias()); - - BaseConfig transformerConfig = new BaseConfig(SQL,config); + Map inputParameterValueResult, + List transformerConfigList, + DqRuleExecuteSql executeSqlDefinition) { + Map config = new HashMap<>(); + config.put(INDEX, index++); + config.put(SQL, + ParameterUtils.convertParameterPlaceholders(executeSqlDefinition.getSql(), inputParameterValueResult)); + config.put(OUTPUT_TABLE, executeSqlDefinition.getTableAlias()); + + BaseConfig transformerConfig = new BaseConfig(SQL, config); transformerConfigList.add(transformerConfig); return index; } public static List getSingleTableCustomSqlTransformerConfigList(int index, - Map inputParameterValueResult) { + Map inputParameterValueResult) { List list = new ArrayList<>(); - Map config = new HashMap<>(); - config.put(INDEX,index + 1); - config.put(SQL,ParameterUtils.convertParameterPlaceholders(inputParameterValueResult.get(STATISTICS_EXECUTE_SQL),inputParameterValueResult)); - config.put(OUTPUT_TABLE,inputParameterValueResult.get(SRC_TABLE)); - inputParameterValueResult.put(STATISTICS_TABLE,inputParameterValueResult.get(SRC_TABLE)); - BaseConfig transformerConfig = new BaseConfig(SQL,config); + Map config = new HashMap<>(); + config.put(INDEX, index + 1); + config.put(SQL, ParameterUtils.convertParameterPlaceholders( + inputParameterValueResult.get(STATISTICS_EXECUTE_SQL), inputParameterValueResult)); + config.put(OUTPUT_TABLE, inputParameterValueResult.get(SRC_TABLE)); + inputParameterValueResult.put(STATISTICS_TABLE, inputParameterValueResult.get(SRC_TABLE)); + BaseConfig transformerConfig = new BaseConfig(SQL, config); list.add(transformerConfig); - return list; + return list; } private static String getCoalesceString(String table, String column) { return "coalesce(" + table + "." + column + ", '')"; } - private static String getSrcColumnIsNullStr(String table,List columns) { + private static String getSrcColumnIsNullStr(String table, List columns) { String[] columnList = new String[columns.size()]; for (int i = 0; i < columns.size(); i++) { String column = columns.get(i); columnList[i] = table + "." + column + " IS NULL"; } - return String.join(AND, columnList); + return String.join(AND, columnList); } - public static Map getInputParameterMapFromEntryList(List defaultInputEntryList) { + public static Map getInputParameterMapFromEntryList(List defaultInputEntryList) { - Map defaultInputParameterValue = new HashMap<>(); - for (DqRuleInputEntry inputEntry:defaultInputEntryList) { - defaultInputParameterValue.put(inputEntry.getField(),inputEntry.getValue()); + Map defaultInputParameterValue = new HashMap<>(); + for (DqRuleInputEntry inputEntry : defaultInputEntryList) { + defaultInputParameterValue.put(inputEntry.getField(), inputEntry.getValue()); } return defaultInputParameterValue; } public static List getWriterConfigList( - String sql, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + String sql, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List writerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getWriterConnectorType())) { @@ -264,15 +273,17 @@ public class RuleParserUtils { BaseConfig writerConfig = new BaseConfig(); writerConfig.setType(dataQualityTaskExecutionContext.getWriterConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (writerDataSource != null) { - config.put(DATABASE,writerDataSource.getDatabase()); - config.put(TABLE,dataQualityTaskExecutionContext.getWriterTable()); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()),writerDataSource)); - config.put(USER,writerDataSource.getUser()); - config.put(PASSWORD,writerDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); - config.put(SQL,sql); + config.put(DATABASE, writerDataSource.getDatabase()); + config.put(TABLE, dataQualityTaskExecutionContext.getWriterTable()); + config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()), + writerDataSource)); + config.put(USER, writerDataSource.getUser()); + config.put(PASSWORD, writerDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); + config.put(SQL, sql); } writerConfig.setConfig(config); writerConfigList.add(writerConfig); @@ -281,46 +292,48 @@ public class RuleParserUtils { return writerConfigList; } - public static void addStatisticsValueTableReaderConfig (List readerConfigList, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { + public static void addStatisticsValueTableReaderConfig(List readerConfigList, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { if (dataQualityTaskExecutionContext.isComparisonNeedStatisticsValueTable()) { - List statisticsBaseConfigList = RuleParserUtils.getStatisticsValueConfigReaderList(dataQualityTaskExecutionContext); + List statisticsBaseConfigList = + RuleParserUtils.getStatisticsValueConfigReaderList(dataQualityTaskExecutionContext); readerConfigList.addAll(statisticsBaseConfigList); } } - public static List getStatisticsValueConfigWriterList ( - String sql, - Map inputParameterValueResult, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + public static List getStatisticsValueConfigWriterList( + String sql, + Map inputParameterValueResult, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List writerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConfig writerConfig = getStatisticsValueConfig(dataQualityTaskExecutionContext); if (writerConfig != null) { - writerConfig.getConfig().put(SQL,ParameterUtils.convertParameterPlaceholders(sql,inputParameterValueResult)); + writerConfig.getConfig().put(SQL, + ParameterUtils.convertParameterPlaceholders(sql, inputParameterValueResult)); } writerConfigList.add(writerConfig); } return writerConfigList; } - public static List getStatisticsValueConfigReaderList ( - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + public static List getStatisticsValueConfigReaderList( + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List readerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConfig readerConfig = getStatisticsValueConfig(dataQualityTaskExecutionContext); if (readerConfig != null) { - readerConfig.getConfig().put(OUTPUT_TABLE,dataQualityTaskExecutionContext.getStatisticsValueTable()); + readerConfig.getConfig().put(OUTPUT_TABLE, dataQualityTaskExecutionContext.getStatisticsValueTable()); } readerConfigList.add(readerConfig); } return readerConfigList; } - public static BaseConfig getStatisticsValueConfig ( - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + public static BaseConfig getStatisticsValueConfig( + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { BaseConfig baseConfig = null; if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConnectionParam writerDataSource = @@ -330,14 +343,16 @@ public class RuleParserUtils { baseConfig = new BaseConfig(); baseConfig.setType(dataQualityTaskExecutionContext.getStatisticsValueConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (writerDataSource != null) { - config.put(DATABASE,writerDataSource.getDatabase()); - config.put(TABLE,dataQualityTaskExecutionContext.getStatisticsValueTable()); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()),writerDataSource)); - config.put(USER,writerDataSource.getUser()); - config.put(PASSWORD,writerDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); + config.put(DATABASE, writerDataSource.getDatabase()); + config.put(TABLE, dataQualityTaskExecutionContext.getStatisticsValueTable()); + config.put(URL, DataSourceUtils.getJdbcUrl( + DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()), writerDataSource)); + config.put(USER, writerDataSource.getUser()); + config.put(PASSWORD, writerDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); } baseConfig.setConfig(config); @@ -346,45 +361,53 @@ public class RuleParserUtils { return baseConfig; } - public static String getOnClause(List mappingColumnList,Map inputParameterValueResult) { - //get on clause + public static String getOnClause(List mappingColumnList, + Map inputParameterValueResult) { + // get on clause String[] columnList = new String[mappingColumnList.size()]; for (int i = 0; i < mappingColumnList.size(); i++) { MappingColumn column = mappingColumnList.get(i); - columnList[i] = getCoalesceString(inputParameterValueResult.get(SRC_TABLE),column.getSrcField()) + columnList[i] = getCoalesceString(inputParameterValueResult.get(SRC_TABLE), column.getSrcField()) + column.getOperator() - + getCoalesceString(inputParameterValueResult.get(TARGET_TABLE),column.getTargetField()); + + getCoalesceString(inputParameterValueResult.get(TARGET_TABLE), column.getTargetField()); } - return String.join(AND,columnList); + return String.join(AND, columnList); } - public static String getWhereClause(List mappingColumnList,Map inputParameterValueResult) { - String srcColumnNotNull = "( NOT (" + getSrcColumnIsNullStr(inputParameterValueResult.get(SRC_TABLE),getSrcColumnList(mappingColumnList)) + " ))"; - String targetColumnIsNull = "( " + getSrcColumnIsNullStr(inputParameterValueResult.get(TARGET_TABLE),getTargetColumnList(mappingColumnList)) + " )"; + public static String getWhereClause(List mappingColumnList, + Map inputParameterValueResult) { + String srcColumnNotNull = "( NOT (" + + getSrcColumnIsNullStr(inputParameterValueResult.get(SRC_TABLE), getSrcColumnList(mappingColumnList)) + + " ))"; + String targetColumnIsNull = "( " + getSrcColumnIsNullStr(inputParameterValueResult.get(TARGET_TABLE), + getTargetColumnList(mappingColumnList)) + " )"; return srcColumnNotNull + AND + targetColumnIsNull; } public static List getWriterConfigList( - int index, - Map inputParameterValueResult, - List transformerConfigList, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext, - String writerSql) throws DataQualityException { + int index, + Map inputParameterValueResult, + List transformerConfigList, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext, + String writerSql) throws DataQualityException { List comparisonExecuteSqlList = - getExecuteSqlListByType(JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList(),DqRuleExecuteSql.class), ExecuteSqlType.COMPARISON); + getExecuteSqlListByType( + JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList(), DqRuleExecuteSql.class), + ExecuteSqlType.COMPARISON); if (CollectionUtils.isNotEmpty(comparisonExecuteSqlList)) { DqRuleExecuteSql comparisonSql = comparisonExecuteSqlList.get(0); - inputParameterValueResult.put(COMPARISON_TABLE,comparisonSql.getTableAlias()); + inputParameterValueResult.put(COMPARISON_TABLE, comparisonSql.getTableAlias()); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(SRC_FILTER),AND_SRC_FILTER); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(SRC_FILTER),WHERE_SRC_FILTER); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(TARGET_FILTER),AND_TARGET_FILTER); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(TARGET_FILTER),WHERE_TARGET_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(SRC_FILTER), AND_SRC_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(SRC_FILTER), WHERE_SRC_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(TARGET_FILTER), AND_TARGET_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(TARGET_FILTER), + WHERE_TARGET_FILTER); - for (DqRuleExecuteSql executeSqlDefinition:comparisonExecuteSqlList) { + for (DqRuleExecuteSql executeSqlDefinition : comparisonExecuteSqlList) { index = setTransformerConfig( index, inputParameterValueResult, @@ -394,18 +417,17 @@ public class RuleParserUtils { } return getWriterConfigList( - ParameterUtils.convertParameterPlaceholders(writerSql,inputParameterValueResult), - dataQualityTaskExecutionContext - ); + ParameterUtils.convertParameterPlaceholders(writerSql, inputParameterValueResult), + dataQualityTaskExecutionContext); } - public static List getAllWriterConfigList ( - Map inputParameterValue, - DataQualityTaskExecutionContext context, - int index, - List transformerConfigList, - String writerSql, - String statisticsValueWriterSql) { + public static List getAllWriterConfigList( + Map inputParameterValue, + DataQualityTaskExecutionContext context, + int index, + List transformerConfigList, + String writerSql, + String statisticsValueWriterSql) { List writerConfigList = RuleParserUtils.getWriterConfigList( index, @@ -420,7 +442,7 @@ public class RuleParserUtils { inputParameterValue, context)); - BaseConfig errorOutputWriter = RuleParserUtils.getErrorOutputWriter(inputParameterValue,context); + BaseConfig errorOutputWriter = RuleParserUtils.getErrorOutputWriter(inputParameterValue, context); if (errorOutputWriter != null) { writerConfigList.add(errorOutputWriter); } @@ -428,8 +450,9 @@ public class RuleParserUtils { return writerConfigList; } - public static List getExecuteSqlListByType ( - List allExecuteSqlList, ExecuteSqlType executeSqlType) { + public static List getExecuteSqlListByType( + List allExecuteSqlList, + ExecuteSqlType executeSqlType) { if (CollectionUtils.isEmpty(allExecuteSqlList)) { return allExecuteSqlList; } @@ -442,9 +465,9 @@ public class RuleParserUtils { private static void checkAndReplace(List list, String checkValue, String replaceSrc) { if (StringUtils.isEmpty(checkValue) && CollectionUtils.isNotEmpty(list)) { - for (DqRuleExecuteSql executeSqlDefinition:list) { + for (DqRuleExecuteSql executeSqlDefinition : list) { String sql = executeSqlDefinition.getSql(); - sql = sql.replace(replaceSrc,""); + sql = sql.replace(replaceSrc, ""); executeSqlDefinition.setSql(sql); } } @@ -455,9 +478,9 @@ public class RuleParserUtils { List list = new ArrayList<>(); mappingColumnList.forEach(item -> { MappingColumn column = new MappingColumn( - String.valueOf(item.get(SRC_FIELD)).replace("\"",""), - String.valueOf(item.get(OPERATOR)).replace("\""," "), - String.valueOf(item.get(TARGET_FIELD)).replace("\"","")); + String.valueOf(item.get(SRC_FIELD)).replace("\"", ""), + String.valueOf(item.get(OPERATOR)).replace("\"", " "), + String.valueOf(item.get(TARGET_FIELD)).replace("\"", "")); list.add(column); }); @@ -466,18 +489,14 @@ public class RuleParserUtils { public static List getSrcColumnList(List mappingColumns) { List list = new ArrayList<>(); - mappingColumns.forEach(item -> - list.add(item.getSrcField()) - ); + mappingColumns.forEach(item -> list.add(item.getSrcField())); return list; } public static List getTargetColumnList(List mappingColumns) { List list = new ArrayList<>(); - mappingColumns.forEach(item -> - list.add(item.getTargetField()) - ); + mappingColumns.forEach(item -> list.add(item.getTargetField())); return list; } @@ -485,13 +504,13 @@ public class RuleParserUtils { public static EnvConfig getEnvConfig() { EnvConfig envConfig = new EnvConfig(); envConfig.setType(BATCH); - return envConfig; + return envConfig; } public static BaseConfig getErrorOutputWriter(Map inputParameterValueResult, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { - List dqRuleExecuteSqlList = JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList() - ,DqRuleExecuteSql.class); + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { + List dqRuleExecuteSqlList = + JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList(), DqRuleExecuteSql.class); DqRuleExecuteSql errorOutputSql = null; if (CollectionUtils.isEmpty(dqRuleExecuteSqlList)) { @@ -509,9 +528,9 @@ public class RuleParserUtils { if (StringUtils.isNotEmpty(inputParameterValueResult.get(ERROR_OUTPUT_PATH)) && errorOutputSql != null) { baseConfig = new BaseConfig(); - Map config = new HashMap<>(); - config.put(PATH,inputParameterValueResult.get(ERROR_OUTPUT_PATH)); - config.put(INPUT_TABLE,errorOutputSql.getTableAlias()); + Map config = new HashMap<>(); + config.put(PATH, inputParameterValueResult.get(ERROR_OUTPUT_PATH)); + config.put(INPUT_TABLE, errorOutputSql.getTableAlias()); baseConfig.setConfig(config); baseConfig.setType(HDFS_FILE); } @@ -530,7 +549,7 @@ public class RuleParserUtils { return "-1"; } - Map newInputParameterValue = new HashMap<>(inputParameterValue); + Map newInputParameterValue = new HashMap<>(inputParameterValue); newInputParameterValue.remove(RULE_TYPE); newInputParameterValue.remove(RULE_NAME); @@ -557,6 +576,6 @@ public class RuleParserUtils { sb.append(value); } - return Md5Utils.getMd5(sb.toString(),true); + return Md5Utils.getMd5(sb.toString(), true); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java index 4b9260326e..4bd824226b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.plugin.task.dq.utils; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.ProgramType; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.SparkConstants; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.SparkParameters; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -103,7 +103,8 @@ public class SparkArgsUtils { } String others = param.getOthers(); - if (!SPARK_LOCAL.equals(deployMode) && (StringUtils.isEmpty(others) || !others.contains(SparkConstants.SPARK_QUEUE))) { + if (!SPARK_LOCAL.equals(deployMode) + && (StringUtils.isEmpty(others) || !others.contains(SparkConstants.SPARK_QUEUE))) { String queue = param.getQueue(); if (StringUtils.isNotEmpty(queue)) { args.add(SparkConstants.SPARK_QUEUE); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml index 3aa7bcacb1..b3e1edb79f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-datasync jar @@ -51,4 +50,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java index 21e271ad7d..261c4a703b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java @@ -17,15 +17,17 @@ package org.apache.dolphinscheduler.plugin.task.datasync; -import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.List; + import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.spi.utils.StringUtils; -import java.util.List; +import com.fasterxml.jackson.annotation.JsonProperty; @Setter @Getter @@ -56,7 +58,8 @@ public class DatasyncParameters extends AbstractParameters { if (jsonFormat) { return StringUtils.isNotEmpty(json); } else { - return StringUtils.isNotEmpty(destinationLocationArn) && StringUtils.isNotEmpty(sourceLocationArn) && StringUtils.isNotEmpty(name); + return StringUtils.isNotEmpty(destinationLocationArn) && StringUtils.isNotEmpty(sourceLocationArn) + && StringUtils.isNotEmpty(name); } } @@ -64,7 +67,8 @@ public class DatasyncParameters extends AbstractParameters { @Getter @NoArgsConstructor @ToString - public class Options{ + public class Options { + private String verifyMode; private String overwriteMode; private String atime; @@ -85,14 +89,16 @@ public class DatasyncParameters extends AbstractParameters { @Getter @NoArgsConstructor @ToString - public static class TaskSchedule{ + public static class TaskSchedule { + private String scheduleExpression; } @Setter @Getter @NoArgsConstructor @ToString - public static class FilterRule{ + public static class FilterRule { + private String filterType; private String value; } @@ -100,8 +106,9 @@ public class DatasyncParameters extends AbstractParameters { @Getter @NoArgsConstructor @ToString - public static class TagListEntry{ + public static class TagListEntry { + private String key; private String value; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java index 555eda934e..3890f6d641 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java @@ -45,4 +45,4 @@ public class DatasyncTaskChannel implements TaskChannel { public ResourceParametersHelper getResources(String parameters) { return null; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java index a14b24f13f..5fac501bea 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.datasync; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -25,6 +24,8 @@ import org.apache.dolphinscheduler.spi.params.base.PluginParams; import java.util.Collections; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(TaskChannelFactory.class) public class DatasyncTaskChannelFactory implements TaskChannelFactory { @@ -42,4 +43,4 @@ public class DatasyncTaskChannelFactory implements TaskChannelFactory { public List getParams() { return Collections.emptyList(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index 02b4428e86..9cae62ad94 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-datax jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java index 391e94658e..e90e5a86be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.spi.enums.Flag; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -288,8 +287,10 @@ public class DataxParameters extends AbstractParameters { return dataxTaskExecutionContext; } - DataSourceParameters dbSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataSource); - DataSourceParameters dbTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataTarget); + DataSourceParameters dbSource = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataSource); + DataSourceParameters dbTarget = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataTarget); if (Objects.nonNull(dbSource)) { dataxTaskExecutionContext.setDataSourceId(dataSource); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 48a7f7f743..7df406f6a2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -17,26 +17,15 @@ package org.apache.dolphinscheduler.plugin.task.datax; -import com.alibaba.druid.sql.ast.SQLStatement; -import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; -import com.alibaba.druid.sql.ast.statement.SQLSelect; -import com.alibaba.druid.sql.ast.statement.SQLSelectItem; -import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; -import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; -import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; -import com.alibaba.druid.sql.parser.SQLStatementParser; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; + import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; @@ -50,6 +39,10 @@ import org.apache.dolphinscheduler.spi.enums.Flag; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; + import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -72,11 +65,20 @@ import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; public class DataxTask extends AbstractTask { + /** * jvm parameters */ @@ -139,7 +141,8 @@ public class DataxTask extends AbstractTask { throw new RuntimeException("datax task params is not valid"); } - dataxTaskExecutionContext = dataXParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + dataxTaskExecutionContext = + dataXParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); } /** @@ -193,8 +196,7 @@ public class DataxTask extends AbstractTask { * @return datax json file name * @throws Exception if error throws Exception */ - private String buildDataxJsonFile(Map paramsMap) - throws Exception { + private String buildDataxJsonFile(Map paramsMap) throws Exception { // generate json String fileName = String.format("%s/%s_job.json", taskExecutionContext.getExecutePath(), @@ -271,7 +273,8 @@ public class DataxTask extends AbstractTask { ArrayNode tableArr = writerConn.putArray("table"); tableArr.add(dataXParameters.getTargetTable()); - writerConn.put("jdbcUrl", DataSourceUtils.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataTargetCfg)); + writerConn.put("jdbcUrl", + DataSourceUtils.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataTargetCfg)); writerConnArr.add(writerConn); ObjectNode writerParam = JSONUtils.createObjectNode(); @@ -377,8 +380,7 @@ public class DataxTask extends AbstractTask { * @return shell command file name * @throws Exception if error throws Exception */ - private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) - throws Exception { + private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) throws Exception { // generate scripts String fileName = String.format("%s/%s_node.%s", taskExecutionContext.getExecutePath(), @@ -466,7 +468,8 @@ public class DataxTask extends AbstractTask { * @param sql sql for data synchronization * @return Keyword converted column names */ - private String[] parsingSqlColumnNames(DbType sourceType, DbType targetType, BaseConnectionParam dataSourceCfg, String sql) { + private String[] parsingSqlColumnNames(DbType sourceType, DbType targetType, BaseConnectionParam dataSourceCfg, + String sql) { String[] columnNames = tryGrammaticalAnalysisSqlColumnNames(sourceType, sql); if (columnNames == null || columnNames.length == 0) { @@ -563,7 +566,8 @@ public class DataxTask extends AbstractTask { sql = sql.replace(";", ""); try ( - Connection connection = DataSourceClientProvider.getInstance().getConnection(sourceType, baseDataSource); + Connection connection = + DataSourceClientProvider.getInstance().getConnection(sourceType, baseDataSource); PreparedStatement stmt = connection.prepareStatement(sql); ResultSet resultSet = stmt.executeQuery()) { @@ -571,7 +575,7 @@ public class DataxTask extends AbstractTask { int num = md.getColumnCount(); columnNames = new String[num]; for (int i = 1; i <= num; i++) { - columnNames[i - 1] = md.getColumnName(i).replace("t.",""); + columnNames[i - 1] = md.getColumnName(i).replace("t.", ""); } } catch (SQLException | ExecutionException e) { logger.error(e.getMessage(), e); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java index d2c4b1d67e..9ecb0bbb86 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.datax; -import com.alibaba.druid.sql.dialect.presto.parser.PrestoStatementParser; import org.apache.dolphinscheduler.spi.enums.DbType; import com.alibaba.druid.sql.dialect.clickhouse.parser.ClickhouseStatementParser; @@ -25,6 +24,7 @@ import com.alibaba.druid.sql.dialect.hive.parser.HiveStatementParser; import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import com.alibaba.druid.sql.dialect.presto.parser.PrestoStatementParser; import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser; import com.alibaba.druid.sql.parser.SQLStatementParser; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java index 636ba839c0..77b89e65ab 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java @@ -17,7 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.datax; -import org.apache.commons.lang3.SystemUtils; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; @@ -33,13 +38,8 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; + +import org.apache.commons.lang3.SystemUtils; import java.io.File; import java.io.IOException; @@ -54,19 +54,21 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.Map; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class DataxTaskTest { private DataxTask dataxTask; - private final TaskCallBack taskCallBack = (taskInstanceId, appIds) -> {}; + private final TaskCallBack taskCallBack = (taskInstanceId, appIds) -> { + }; @BeforeEach public void before() throws Exception { @@ -109,9 +111,8 @@ public class DataxTaskTest { boolean delete = jsonFile.delete(); Assertions.assertTrue(delete); - File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? - new File("/tmp/execution/app-id_node.bat") : - new File("/tmp/execution/app-id_node.sh"); + File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? new File("/tmp/execution/app-id_node.bat") + : new File("/tmp/execution/app-id_node.sh"); InputStream shellCommandInputStream = Files.newInputStream(shellCommandFile.toPath()); String shellCommandStr = FileUtils.readFile2Str(shellCommandInputStream); Assertions.assertEquals(shellCommandStr, "python2.7 ${DATAX_HOME}/bin/datax.py --jvm=\"-Xms1G -Xmx1G\" " + @@ -151,9 +152,8 @@ public class DataxTaskTest { boolean delete = jsonFile.delete(); Assertions.assertTrue(delete); - File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? - new File("/tmp/execution/app-id_node.bat") : - new File("/tmp/execution/app-id_node.sh"); + File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? new File("/tmp/execution/app-id_node.bat") + : new File("/tmp/execution/app-id_node.sh"); InputStream shellCommandInputStream = Files.newInputStream(shellCommandFile.toPath()); String shellCommandStr = FileUtils.readFile2Str(shellCommandInputStream); Assertions.assertEquals(shellCommandStr, "python2.7 ${DATAX_HOME}/bin/datax.py --jvm=\"-Xms1G -Xmx1G\" " + @@ -222,12 +222,12 @@ public class DataxTaskTest { when(resultSet.getMetaData()).thenReturn(md); when(stmt.executeQuery()).thenReturn(resultSet); - String[] rows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL,baseConnectionParam, ""); + String[] rows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL, baseConnectionParam, ""); Assertions.assertEquals(rows.length, 1); Assertions.assertEquals(rows[0], "something"); when(connection.prepareStatement(anyString())).thenThrow(new SQLException("Connection failed")); - String[] nullRows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL,baseConnectionParam, ""); + String[] nullRows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL, baseConnectionParam, ""); Assertions.assertNull(nullRows); } } @@ -324,4 +324,4 @@ public class DataxTaskTest { " }\n" + "}"; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml index fc7a4eb286..6fe7635444 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dependent jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java index 1d24e8527a..1d7e62422a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java @@ -42,7 +42,8 @@ public class DependentTaskChannel implements TaskChannel { @Override public AbstractParameters parseParameters(ParametersNode parametersNode) { return JSONUtils.parseObject(StringUtils.isEmpty(parametersNode.getDependence()) - ? parametersNode.getTaskParams() : parametersNode.getDependence(), DependentParameters.class); + ? parametersNode.getTaskParams() + : parametersNode.getDependence(), DependentParameters.class); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java index ad976d5857..4d323f3388 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java @@ -29,6 +29,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class DependentTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new DependentTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml index ec3259c0bc..647fbfab5b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml @@ -15,18 +15,17 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 - jar dolphinscheduler-task-dinky + jar org.apache.dolphinscheduler @@ -47,4 +46,4 @@ httpcore - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java index 2b0d6c1d32..2349a39b5a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java @@ -72,9 +72,9 @@ public class DinkyParameters extends AbstractParameters { @Override public String toString() { return "DinkyParameters{" + - "address='" + address + '\'' + - ", taskId='" + taskId + '\'' + - ", online='" + online + '\'' + - '}'; + "address='" + address + '\'' + + ", taskId='" + taskId + '\'' + + ", online='" + online + '\'' + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java index 6c80a9769d..ddaa53d233 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.dinky; -import com.google.auto.service.AutoService; - import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -26,8 +24,11 @@ import org.apache.dolphinscheduler.spi.params.base.PluginParams; import java.util.ArrayList; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(TaskChannelFactory.class) public class DinkyTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "DINKY"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java index 847d4e3a31..8b12b8c259 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java @@ -25,14 +25,15 @@ public class DinkyTaskConstants { private DinkyTaskConstants() { throw new IllegalStateException("Utility class"); } - + private static final String API_ROUTE = "/openapi/"; public static final String SUBMIT_TASK = API_ROUTE + "submitTask"; public static final String ONLINE_TASK = API_ROUTE + "onLineTask"; public static final String SAVEPOINT_TASK = API_ROUTE + "savepointTask"; public static final String GET_JOB_INFO = API_ROUTE + "getJobInstance"; public static final int API_ERROR = 1; - public static final String API_VERSION_ERROR_TIPS = "Please check that the dinky version is greater than or equal to 0.6.5"; + public static final String API_VERSION_ERROR_TIPS = + "Please check that the dinky version is greater than or equal to 0.6.5"; public static final String API_RESULT_DATAS = "datas"; public static final String SAVEPOINT_CANCEL = "cancel"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml index e291594a08..8a3bb85413 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dms jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java index de30df70c5..c53dac3358 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java @@ -22,10 +22,10 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters import java.util.Date; import java.util.List; -import com.amazonaws.services.databasemigrationservice.model.Tag; - import lombok.Data; +import com.amazonaws.services.databasemigrationservice.model.Tag; + @Data public class DmsParameters extends AbstractParameters { @@ -57,7 +57,7 @@ public class DmsParameters extends AbstractParameters { flag = replicationTaskArn != null; } else { flag = sourceEndpointArn != null && targetEndpointArn != null && replicationInstanceArn != null - && migrationType != null && replicationTaskIdentifier != null && tableMappings != null; + && migrationType != null && replicationTaskIdentifier != null && tableMappings != null; } return flag; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java index 177ca9bf19..a54e0aad46 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java @@ -44,11 +44,11 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategy; public class DmsTask extends AbstractRemoteTask { private static final ObjectMapper objectMapper = - new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false) - .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) - .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) - .configure(REQUIRE_SETTERS_FOR_GETTERS, true) - .setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); + new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) + .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); /** * taskExecutionContext */ @@ -102,7 +102,8 @@ public class DmsTask extends AbstractRemoteTask { dmsHook.setReplicationTaskArn(appId.getReplicationTaskArn()); // if CdcStopPosition is not set, the task will not continue to check the running status if (isStopTaskWhenCdc()) { - logger.info("This is a cdc task and cdcStopPosition is not set, the task will not continue to check the running status"); + logger.info( + "This is a cdc task and cdcStopPosition is not set, the task will not continue to check the running status"); exitStatusCode = TaskConstants.EXIT_CODE_SUCCESS; return; } @@ -166,7 +167,8 @@ public class DmsTask extends AbstractRemoteTask { } catch (InvalidResourceStateException e) { logger.error("Failed to start a task, error message: {}", e.getErrorMessage()); - // Only restart task when the error contains "Test connection", means instance can not connect to source or target + // Only restart task when the error contains "Test connection", means instance can not connect to source or + // target if (!e.getErrorMessage().contains("Test connection")) { return TaskConstants.EXIT_CODE_FAILURE; } @@ -231,7 +233,8 @@ public class DmsTask extends AbstractRemoteTask { // create a new parameter object using the json data if the json data is not empty if (parameters.getIsJsonFormat() && parameters.getJsonData() != null) { // combining local and global parameters - String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); + String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), + ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); boolean isRestartTask = parameters.getIsRestartTask(); try { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java index addc00a028..a8564ff8a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.apache.dolphinscheduler.plugin.task.dms; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml index a0435647ca..93825996f4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dvc jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java index 6f65bedc74..d43ecbedd5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java @@ -18,17 +18,20 @@ package org.apache.dolphinscheduler.plugin.task.dvc; public class DvcConstants { + private DvcConstants() { throw new IllegalStateException("Utility class"); } public static final class DVC_TASK_TYPE { + public static final String UPLOAD = "Upload"; public static final String DOWNLOAD = "Download"; public static final String INIT = "Init DVC"; }; - public static final String CHECK_AND_SET_DVC_REPO = "which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=%s"; + public static final String CHECK_AND_SET_DVC_REPO = + "which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=%s"; public static final String SET_DATA_PATH = "DVC_DATA_PATH=%s"; @@ -38,19 +41,19 @@ public class DvcConstants { public static final String SET_MESSAGE = "DVC_MESSAGE=\"%s\""; - public static final String GIT_CLONE_DVC_REPO = "git clone $DVC_REPO dvc-repository; cd dvc-repository; pwd"; + public static final String GIT_CLONE_DVC_REPO = "git clone $DVC_REPO dvc-repository; cd dvc-repository; pwd"; - public static final String DVC_AUTOSTAGE = "dvc config core.autostage true --local || exit 1"; + public static final String DVC_AUTOSTAGE = "dvc config core.autostage true --local || exit 1"; - public static final String DVC_ADD_DATA = "dvc add $DVC_DATA_PATH -v -o $DVC_DATA_LOCATION --to-remote || exit 1"; + public static final String DVC_ADD_DATA = "dvc add $DVC_DATA_PATH -v -o $DVC_DATA_LOCATION --to-remote || exit 1"; - public static final String GIT_UPDATE_FOR_UPDATE_DATA = "git commit -am \"$DVC_MESSAGE\"\n" + + public static final String GIT_UPDATE_FOR_UPDATE_DATA = "git commit -am \"$DVC_MESSAGE\"\n" + "git tag \"$DVC_VERSION\" -m \"$DVC_MESSAGE\"\n" + "git push --all\n" + "git push --tags"; - public static final String DVC_DOWNLOAD = "dvc get $DVC_REPO $DVC_DATA_LOCATION -o $DVC_DATA_PATH -v --rev $DVC_VERSION"; - + public static final String DVC_DOWNLOAD = + "dvc get $DVC_REPO $DVC_DATA_LOCATION -o $DVC_DATA_PATH -v --rev $DVC_VERSION"; public static final String DVC_INIT = "dvc init || exit 1"; @@ -59,4 +62,3 @@ public class DvcConstants { public static final String GIT_UPDATE_FOR_INIT_DVC = "git commit -am \"init dvc project and add remote\"; git push"; } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java index ec93e48284..932abf83b7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.task.dvc; -import lombok.Data; - import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; +import lombok.Data; + @Data public class DvcParameters extends AbstractParameters { @@ -53,24 +53,23 @@ public class DvcParameters extends AbstractParameters { switch (dvcTaskType) { case DvcConstants.DVC_TASK_TYPE.UPLOAD: return StringUtils.isNotEmpty(dvcRepository) && - StringUtils.isNotEmpty(dvcDataLocation) && - StringUtils.isNotEmpty(dvcLoadSaveDataPath) && - StringUtils.isNotEmpty(dvcVersion) && - StringUtils.isNotEmpty(dvcMessage); + StringUtils.isNotEmpty(dvcDataLocation) && + StringUtils.isNotEmpty(dvcLoadSaveDataPath) && + StringUtils.isNotEmpty(dvcVersion) && + StringUtils.isNotEmpty(dvcMessage); case DvcConstants.DVC_TASK_TYPE.DOWNLOAD: return StringUtils.isNotEmpty(dvcRepository) && - StringUtils.isNotEmpty(dvcDataLocation) && - StringUtils.isNotEmpty(dvcLoadSaveDataPath) && - StringUtils.isNotEmpty(dvcVersion); + StringUtils.isNotEmpty(dvcDataLocation) && + StringUtils.isNotEmpty(dvcLoadSaveDataPath) && + StringUtils.isNotEmpty(dvcVersion); case DvcConstants.DVC_TASK_TYPE.INIT: return StringUtils.isNotEmpty(dvcRepository) && - StringUtils.isNotEmpty(dvcStoreUrl); + StringUtils.isNotEmpty(dvcStoreUrl); default: return false; } } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java index adccccede7..107b21b3b3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.JSONUtils; - public class DvcTaskChannel implements TaskChannel { @Override @@ -48,4 +47,3 @@ public class DvcTaskChannel implements TaskChannel { } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java index f028d4e0d6..a29ffded5b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java @@ -32,6 +32,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class DvcTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new DvcTaskChannel(); @@ -46,13 +47,15 @@ public class DvcTaskChannelFactory implements TaskChannelFactory { public List getParams() { List paramsList = new ArrayList<>(); - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')").addValidate(Validate.newBuilder().setRequired(true).build()).build(); + InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") + .addValidate(Validate.newBuilder().setRequired(true).build()).build(); - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG").addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)).addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); + RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") + .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) + .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); paramsList.add(nodeName); paramsList.add(runFlag); return paramsList; } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml index 7a4e9913a2..969756a991 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-emr jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java index 3adce9e1b5..c6be686347 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java @@ -21,6 +21,7 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; + import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java index 1e38a66d81..8b045d6edc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java @@ -55,10 +55,11 @@ public class EmrParameters extends AbstractParameters { public boolean checkParameters() { /* * When saving a task, the programType cannot be empty and jobFlowDefineJson or stepsDefineJson cannot be empty: - * (1) When ProgramType is RUN_JOB_FLOW, jobFlowDefineJson cannot be empty. - * (2) When ProgramType is ADD_JOB_FLOW_STEPS, stepsDefineJson cannot be empty. + * (1) When ProgramType is RUN_JOB_FLOW, jobFlowDefineJson cannot be empty. (2) When ProgramType is + * ADD_JOB_FLOW_STEPS, stepsDefineJson cannot be empty. */ - return programType != null && (StringUtils.isNotEmpty(jobFlowDefineJson) || StringUtils.isNotEmpty(stepsDefineJson)); + return programType != null + && (StringUtils.isNotEmpty(jobFlowDefineJson) || StringUtils.isNotEmpty(stepsDefineJson)); } @Override @@ -70,9 +71,9 @@ public class EmrParameters extends AbstractParameters { @Override public String toString() { return "EmrParameters{" - + "programType=" + programType - + ", jobFlowDefineJson='" + jobFlowDefineJson + '\'' - + ", stepsDefineJson='" + stepsDefineJson + '\'' - + '}'; + + "programType=" + programType + + ", jobFlowDefineJson='" + jobFlowDefineJson + '\'' + + ", stepsDefineJson='" + stepsDefineJson + '\'' + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java index e59b4e3613..812f7ed121 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class EmrTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { // no need diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java index daf67cb2d2..cad5583f07 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java @@ -28,6 +28,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class EmrTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "EMR"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml index e2dd5753f8..2cd1a9f4f2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-flink-stream jar @@ -45,4 +44,4 @@ ${project.version} - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java index 5a86dbe3f2..2bf37e326c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; -import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; @@ -27,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.task.api.stream.StreamTaskChannel; import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class FlinkStreamTaskChannel implements StreamTaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java index f90a1e47b7..475386bf36 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class FlinkStreamTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new FlinkStreamTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml index c43d507d3c..5a208a89ac 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml @@ -15,29 +15,28 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-flink jar - - org.apache.dolphinscheduler - dolphinscheduler-spi - provided - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - ${project.version} - + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java index 33f0fecbfb..4672fad31b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.flink; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; + import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.SystemUtils; import java.io.File; import java.io.IOException; @@ -34,18 +35,23 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); - private FileUtils() {} + private FileUtils() { + } public static String getInitScriptFilePath(TaskExecutionContext taskExecutionContext) { - return String.format("%s/%s_init.sql", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); + return String.format("%s/%s_init.sql", taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); } public static String getScriptFilePath(TaskExecutionContext taskExecutionContext) { - return String.format("%s/%s_node.sql", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); + return String.format("%s/%s_node.sql", taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); } public static void generateScriptFile(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { @@ -53,8 +59,7 @@ public class FileUtils { String scriptFilePath = FileUtils.getScriptFilePath(taskExecutionContext); String initOptionsString = StringUtils.join( FlinkArgsUtils.buildInitOptionsForSql(flinkParameters), - FlinkConstants.FLINK_SQL_NEWLINE - ).concat(FlinkConstants.FLINK_SQL_NEWLINE); + FlinkConstants.FLINK_SQL_NEWLINE).concat(FlinkConstants.FLINK_SQL_NEWLINE); writeScriptFile(initScriptFilePath, initOptionsString + flinkParameters.getInitScript()); writeScriptFile(scriptFilePath, flinkParameters.getRawScript()); } @@ -66,7 +71,8 @@ public class FileUtils { try { Files.delete(path); } catch (IOException e) { - throw new RuntimeException(String.format("Flink Script file exists in path: %s before creation and cannot be deleted", path), e); + throw new RuntimeException(String + .format("Flink Script file exists in path: %s before creation and cannot be deleted", path), e); } } @@ -96,7 +102,7 @@ public class FileUtils { LOGGER.info("Writing content: " + content); LOGGER.info("To file: " + file.getAbsolutePath()); Files.write(file.getAbsoluteFile().toPath(), content.getBytes(StandardCharsets.UTF_8), standardOpenOption); - } catch(IOException e) { + } catch (IOException e) { throw new RuntimeException("Error writing file: " + file.getAbsoluteFile(), e); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java index 403a9bc632..9508008eb0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java @@ -24,8 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -95,7 +93,8 @@ public class FlinkArgsUtils { * * @return argument list */ - private static List buildRunCommandLineForSql(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { + private static List buildRunCommandLineForSql(TaskExecutionContext taskExecutionContext, + FlinkParameters flinkParameters) { List args = new ArrayList<>(); args.add(FlinkConstants.FLINK_SQL_COMMAND); @@ -120,7 +119,8 @@ public class FlinkArgsUtils { public static List buildInitOptionsForSql(FlinkParameters flinkParameters) { List initOptions = new ArrayList<>(); - FlinkDeployMode deployMode = Optional.ofNullable(flinkParameters.getDeployMode()).orElse(FlinkDeployMode.CLUSTER); + FlinkDeployMode deployMode = + Optional.ofNullable(flinkParameters.getDeployMode()).orElse(FlinkDeployMode.CLUSTER); /** * Currently flink sql on yarn only supports yarn-per-job mode @@ -130,7 +130,8 @@ public class FlinkArgsUtils { initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, "local")); } else { // execution.target - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, FlinkConstants.FLINK_YARN_PER_JOB)); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, FlinkConstants.FLINK_YARN_PER_JOB)); // taskmanager.numberOfTaskSlots int slot = flinkParameters.getSlot(); @@ -141,19 +142,22 @@ public class FlinkArgsUtils { // yarn.application.name String appName = flinkParameters.getAppName(); if (StringUtils.isNotEmpty(appName)) { - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_YARN_APPLICATION_NAME, ArgsUtils.escape(appName))); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_YARN_APPLICATION_NAME, ArgsUtils.escape(appName))); } // jobmanager.memory.process.size String jobManagerMemory = flinkParameters.getJobManagerMemory(); if (StringUtils.isNotEmpty(jobManagerMemory)) { - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_JOBMANAGER_MEMORY_PROCESS_SIZE, jobManagerMemory)); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_JOBMANAGER_MEMORY_PROCESS_SIZE, jobManagerMemory)); } // taskmanager.memory.process.size String taskManagerMemory = flinkParameters.getTaskManagerMemory(); if (StringUtils.isNotEmpty(taskManagerMemory)) { - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_TASKMANAGER_MEMORY_PROCESS_SIZE, taskManagerMemory)); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_TASKMANAGER_MEMORY_PROCESS_SIZE, taskManagerMemory)); } // yarn.application.queue @@ -175,7 +179,8 @@ public class FlinkArgsUtils { return initOptions; } - private static List buildRunCommandLineForOthers(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { + private static List buildRunCommandLineForOthers(TaskExecutionContext taskExecutionContext, + FlinkParameters flinkParameters) { List args = new ArrayList<>(); args.add(FlinkConstants.FLINK_COMMAND); @@ -184,23 +189,24 @@ public class FlinkArgsUtils { // build run command switch (deployMode) { case CLUSTER: - if (FLINK_VERSION_AFTER_OR_EQUALS_1_12.equals(flinkVersion) || FLINK_VERSION_AFTER_OR_EQUALS_1_13.equals(flinkVersion)) { - args.add(FlinkConstants.FLINK_RUN); //run - args.add(FlinkConstants.FLINK_EXECUTION_TARGET); //-t - args.add(FlinkConstants.FLINK_YARN_PER_JOB); //yarn-per-job + if (FLINK_VERSION_AFTER_OR_EQUALS_1_12.equals(flinkVersion) + || FLINK_VERSION_AFTER_OR_EQUALS_1_13.equals(flinkVersion)) { + args.add(FlinkConstants.FLINK_RUN); // run + args.add(FlinkConstants.FLINK_EXECUTION_TARGET); // -t + args.add(FlinkConstants.FLINK_YARN_PER_JOB); // yarn-per-job } else { - args.add(FlinkConstants.FLINK_RUN); //run - args.add(FlinkConstants.FLINK_RUN_MODE); //-m - args.add(FlinkConstants.FLINK_YARN_CLUSTER); //yarn-cluster + args.add(FlinkConstants.FLINK_RUN); // run + args.add(FlinkConstants.FLINK_RUN_MODE); // -m + args.add(FlinkConstants.FLINK_YARN_CLUSTER); // yarn-cluster } break; case APPLICATION: - args.add(FlinkConstants.FLINK_RUN_APPLICATION); //run-application - args.add(FlinkConstants.FLINK_EXECUTION_TARGET); //-t - args.add(FlinkConstants.FLINK_YARN_APPLICATION); //yarn-application + args.add(FlinkConstants.FLINK_RUN_APPLICATION); // run-application + args.add(FlinkConstants.FLINK_EXECUTION_TARGET); // -t + args.add(FlinkConstants.FLINK_YARN_APPLICATION); // yarn-application break; case LOCAL: - args.add(FlinkConstants.FLINK_RUN); //run + args.add(FlinkConstants.FLINK_RUN); // run break; } @@ -213,11 +219,11 @@ public class FlinkArgsUtils { int slot = flinkParameters.getSlot(); if (slot > 0) { args.add(FlinkConstants.FLINK_YARN_SLOT); - args.add(String.format("%d", slot)); //-ys + args.add(String.format("%d", slot)); // -ys } String appName = flinkParameters.getAppName(); - if (StringUtils.isNotEmpty(appName)) { //-ynm + if (StringUtils.isNotEmpty(appName)) { // -ynm args.add(FlinkConstants.FLINK_APP_NAME); args.add(ArgsUtils.escape(appName)); } @@ -225,7 +231,7 @@ public class FlinkArgsUtils { // judge flink version, the parameter -yn has removed from flink 1.10 if (flinkVersion == null || FLINK_VERSION_BEFORE_1_10.equals(flinkVersion)) { int taskManager = flinkParameters.getTaskManager(); - if (taskManager > 0) { //-yn + if (taskManager > 0) { // -yn args.add(FlinkConstants.FLINK_TASK_MANAGE); args.add(String.format("%d", taskManager)); } @@ -233,7 +239,7 @@ public class FlinkArgsUtils { String jobManagerMemory = flinkParameters.getJobManagerMemory(); if (StringUtils.isNotEmpty(jobManagerMemory)) { args.add(FlinkConstants.FLINK_JOB_MANAGE_MEM); - args.add(jobManagerMemory); //-yjm + args.add(jobManagerMemory); // -yjm } String taskManagerMemory = flinkParameters.getTaskManagerMemory(); @@ -257,10 +263,11 @@ public class FlinkArgsUtils { int parallelism = flinkParameters.getParallelism(); if (parallelism > 0) { args.add(FlinkConstants.FLINK_PARALLELISM); - args.add(String.format("%d", parallelism)); // -p + args.add(String.format("%d", parallelism)); // -p } - // If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated abruptly + // If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated + // abruptly // The task status will be synchronized with the cluster job status args.add(FlinkConstants.FLINK_SHUTDOWN_ON_ATTACHED_EXIT); // -sae @@ -272,14 +279,14 @@ public class FlinkArgsUtils { ProgramType programType = flinkParameters.getProgramType(); String mainClass = flinkParameters.getMainClass(); if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { - args.add(FlinkConstants.FLINK_MAIN_CLASS); //-c - args.add(flinkParameters.getMainClass()); //main class + args.add(FlinkConstants.FLINK_MAIN_CLASS); // -c + args.add(flinkParameters.getMainClass()); // main class } ResourceInfo mainJar = flinkParameters.getMainJar(); if (mainJar != null) { // -py - if(ProgramType.PYTHON == programType) { + if (ProgramType.PYTHON == programType) { args.add(FlinkConstants.FLINK_PYTHON); } args.add(mainJar.getRes()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java index 2f1daf3bde..e41a1f1a6b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class FlinkTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java index 48363803c5..953f013fce 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class FlinkTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new FlinkTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml index 015d3193e0..fb66bf4706 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml @@ -23,10 +23,10 @@ dolphinscheduler-task-plugin dev-SNAPSHOT - + dolphinscheduler-task-http jar - + org.apache.dolphinscheduler @@ -38,7 +38,7 @@ dolphinscheduler-task-api ${project.version} - + org.apache.commons commons-collections4 @@ -51,7 +51,7 @@ org.apache.httpcomponents httpcore - + com.squareup.okhttp3 mockwebserver diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java index b2f423b22f..479540a1f5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java @@ -27,6 +27,6 @@ public enum HttpCheckCondition { * 2 body_contains * 3 body_not_contains */ - STATUS_CODE_DEFAULT,STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS + STATUS_CODE_DEFAULT, STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java index 2676dd675c..ce76ed1d9a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java @@ -28,6 +28,7 @@ import java.util.List; * http parameter */ public class HttpParameters extends AbstractParameters { + /** * url */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java index 5e0437b698..b027fd874c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java @@ -26,5 +26,5 @@ public enum HttpParametersType { * 1 body; * 2 headers; */ - PARAMETER,BODY,HEADERS + PARAMETER, BODY, HEADERS } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java index 4044b2c57e..5041417b93 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.http; import java.util.Objects; public class HttpProperty { + /** * key */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index 2562d5dcc1..3b16a920f1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -95,13 +95,15 @@ public class HttpTask extends AbstractTask { String statusCode = null; String body = null; - try (CloseableHttpClient client = createHttpClient(); - CloseableHttpResponse response = sendRequest(client)) { + try ( + CloseableHttpClient client = createHttpClient(); + CloseableHttpResponse response = sendRequest(client)) { statusCode = String.valueOf(getStatusCode(response)); body = getResponseBody(response); exitStatusCode = validResponse(body, statusCode); long costTime = System.currentTimeMillis() - startTime; - logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}", + logger.info( + "startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}", formatTimeStamp, httpParameters.getUrl(), httpParameters.getHttpMethod(), costTime, statusCode, body, output); } catch (Exception e) { @@ -141,7 +143,8 @@ public class HttpTask extends AbstractTask { } } addRequestParams(builder, httpPropertyList); - String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap)); + String requestUrl = + ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap)); HttpUriRequest request = builder.setUri(requestUrl).build(); setHeaders(request, httpPropertyList); return client.execute(request); @@ -202,7 +205,8 @@ public class HttpTask extends AbstractTask { break; case STATUS_CODE_CUSTOM: if (!statusCode.equals(httpParameters.getCondition())) { - appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: " + httpParameters.getCondition()); + appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: " + + httpParameters.getCondition()); exitStatusCode = -1; } break; @@ -293,7 +297,8 @@ public class HttpTask extends AbstractTask { * @return RequestConfig */ private RequestConfig requestConfig() { - return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build(); + return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()) + .setConnectTimeout(httpParameters.getConnectTimeout()).build(); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml index 36aa9afeb2..b28effa47b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml @@ -15,16 +15,15 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - - 4.0.0 dolphinscheduler-task-java jar @@ -42,4 +41,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java index e4d941db30..e9bfd581c6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.java; -import java.io.File; - public class JavaConstants { private JavaConstants() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java index 527441b634..de0da08358 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java @@ -27,6 +27,7 @@ import lombok.Data; @Data public class JavaParameters extends AbstractParameters { + /** * origin java script */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index e23eb5124c..3f0f6b2ec5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -125,8 +125,10 @@ public class JavaTask extends AbstractTask { public void handle(TaskCallBack taskCallBack) throws TaskException { try { // Step 1: judge if is java or jar run type. - // Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when building the command - // Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then build the command will add resource + // Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when + // building the command + // Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then + // build the command will add resource // Step 3: to run the command String command = null; switch (javaParameters.getRunType()) { @@ -236,7 +238,6 @@ public class JavaTask extends AbstractTask { return javaParameters; } - /** * Replaces placeholders such as local variables in source files * @@ -280,7 +281,7 @@ public class JavaTask extends AbstractTask { protected void createJavaSourceFileIfNotExists(String sourceCode, String fileName) throws IOException { logger.info("tenantCode: {}, task dir:{}", taskRequest.getTenantCode(), taskRequest.getExecutePath()); if (!Files.exists(Paths.get(fileName))) { - logger.info("the java source code:{}, will be write to the file: {}", fileName,sourceCode); + logger.info("the java source code:{}, will be write to the file: {}", fileName, sourceCode); // write data to file FileUtils.writeStringToFile(new File(fileName), sourceCode, @@ -296,7 +297,8 @@ public class JavaTask extends AbstractTask { * @return String **/ protected String buildJavaSourceCodeFileFullName(String publicClassName) { - return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(), publicClassName); + return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(), + publicClassName); } /** @@ -331,7 +333,7 @@ public class JavaTask extends AbstractTask { **/ protected String buildJavaCompileCommand(String sourceCode) throws IOException { String publicClassName = getPublicClassName(sourceCode); - String fileName = buildJavaSourceCodeFileFullName(publicClassName); + String fileName = buildJavaSourceCodeFileFullName(publicClassName); createJavaSourceFileIfNotExists(sourceCode, fileName); StringBuilder compilerCommand = new StringBuilder() diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java index c374960d45..e72f62e29e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class JavaTaskChannelFactory implements TaskChannelFactory { + /** * Construct a channel for a Java task * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java index 421d77de66..6599ff24cf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.java.exception; public class JavaSourceFileExistException extends RuntimeException { + public JavaSourceFileExistException() { } @@ -33,7 +34,8 @@ public class JavaSourceFileExistException extends RuntimeException { super(cause); } - public JavaSourceFileExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public JavaSourceFileExistException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java index 16b0070c97..103c443378 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.java.exception; public class PublicClassNotFoundException extends RuntimeException { + public PublicClassNotFoundException() { } @@ -33,7 +34,8 @@ public class PublicClassNotFoundException extends RuntimeException { super(cause); } - public PublicClassNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public PublicClassNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml index c81ffc613d..8e23678a13 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-jupyter jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java index 7ecbb29cca..41bff2bf80 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class JupyterTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "JUPYTER"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml index b517bd6bfb..a6b167a218 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml @@ -15,13 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-k8s jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java index a1a613b6af..e6d94fa7ac 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class K8sTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java index dd8caac035..4af560b5d2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class K8sTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "K8S"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml index 7e21ccab77..8e31328390 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-mlflow jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java index a13741230f..c14fb3041c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.mlflow; public class MlflowConstants { + private MlflowConstants() { throw new IllegalStateException("Utility class"); } @@ -59,23 +60,23 @@ public class MlflowConstants { public static final String SET_REPOSITORY = "repo=%s"; public static final String MLFLOW_RUN_BASIC_ALGORITHM = "mlflow run $repo " - + "-P algorithm=%s " - + "-P data_path=$data_path " - + "-P params=\"%s\" " - + "-P search_params=\"%s\" " - + "-P model_name=\"%s\" " - + "--experiment-name=\"%s\""; + + "-P algorithm=%s " + + "-P data_path=$data_path " + + "-P params=\"%s\" " + + "-P search_params=\"%s\" " + + "-P model_name=\"%s\" " + + "--experiment-name=\"%s\""; public static final String MLFLOW_RUN_AUTOML_PROJECT = "mlflow run $repo " - + "-P tool=%s " - + "-P data_path=$data_path " - + "-P params=\"%s\" " - + "-P model_name=\"%s\" " - + "--experiment-name=\"%s\""; + + "-P tool=%s " + + "-P data_path=$data_path " + + "-P params=\"%s\" " + + "-P model_name=\"%s\" " + + "--experiment-name=\"%s\""; public static final String MLFLOW_RUN_CUSTOM_PROJECT = "mlflow run $repo " - + "%s " - + "--experiment-name=\"%s\""; + + "%s " + + "--experiment-name=\"%s\""; public static final String MLFLOW_MODELS_SERVE = "mlflow models serve -m %s --port %s -h 0.0.0.0"; @@ -84,8 +85,8 @@ public class MlflowConstants { public static final String DOCKER_RREMOVE_CONTAINER = "docker rm -f %s"; public static final String DOCKER_RUN = "docker run -d --name=%s -p=%s:8080 " - + "--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20" - + " %s"; + + "--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20" + + " %s"; public static final String DOCKER_HEALTH_CHECK = "docker inspect --format \"{{json .State.Health.Status }}\" %s"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java index 819cb27bd8..61400dc19d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.plugin.task.mlflow; -import lombok.Data; - import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.HashMap; +import lombok.Data; + @Data public class MlflowParameters extends AbstractParameters { @@ -47,7 +47,6 @@ public class MlflowParameters extends AbstractParameters { */ private String automlTool = "FLAML"; - /** * basic algorithm parameters */ @@ -93,7 +92,7 @@ public class MlflowParameters extends AbstractParameters { paramsMap.put("experiment_name", experimentName); paramsMap.put("model_name", modelName); paramsMap.put("MLFLOW_TRACKING_URI", mlflowTrackingUri); - switch (mlflowJobType){ + switch (mlflowJobType) { case MlflowConstants.JOB_TYPE_BASIC_ALGORITHM: addParamsMapForBasicAlgorithm(paramsMap); break; @@ -134,11 +133,11 @@ public class MlflowParameters extends AbstractParameters { return imageName; } - public String getContainerName(){ + public String getContainerName() { return "ds-mlflow-" + getModelKeyName("-"); } - public boolean getIsDeployDocker(){ + public boolean getIsDeployDocker() { if (StringUtils.isEmpty(deployType)) { return false; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java index fbed85e94c..250e8c3b82 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.JSONUtils; - public class MlflowTaskChannel implements TaskChannel { @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java index 5e58fb1915..dc18814d1d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java @@ -32,6 +32,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class MlflowTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new MlflowTaskChannel(); @@ -46,9 +47,12 @@ public class MlflowTaskChannelFactory implements TaskChannelFactory { public List getParams() { List paramsList = new ArrayList<>(); - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')").addValidate(Validate.newBuilder().setRequired(true).build()).build(); + InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") + .addValidate(Validate.newBuilder().setRequired(true).build()).build(); - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG").addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)).addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); + RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") + .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) + .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); paramsList.add(nodeName); paramsList.add(runFlag); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml index 2e00765735..d83329e66a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-mr jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java index e6a67065f4..8a8296fa26 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java @@ -153,7 +153,6 @@ public class MapReduceParameters extends AbstractParameters { + "mainClass=" + mainClass + "mainArgs=" + mainArgs + "queue=" + queue - + "other mainArgs=" + others - ; + + "other mainArgs=" + others; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java index 0a519dc65f..a37d2f9e34 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java @@ -25,11 +25,9 @@ import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -68,7 +66,8 @@ public class MapReduceTask extends AbstractYarnTask { logger.info("mapreduce task params {}", taskExecutionContext.getTaskParams()); - this.mapreduceParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), MapReduceParameters.class); + this.mapreduceParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), MapReduceParameters.class); // check parameters if (mapreduceParameters == null || !mapreduceParameters.checkParameters()) { @@ -81,10 +80,13 @@ public class MapReduceTask extends AbstractYarnTask { // replace placeholder,and combine local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), ParamUtils.convert(paramsMap)); + String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), + ParamUtils.convert(paramsMap)); mapreduceParameters.setMainArgs(args); - if (mapreduceParameters.getProgramType() != null && mapreduceParameters.getProgramType() == ProgramType.PYTHON) { - String others = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getOthers(), ParamUtils.convert(paramsMap)); + if (mapreduceParameters.getProgramType() != null + && mapreduceParameters.getProgramType() == ProgramType.PYTHON) { + String others = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getOthers(), + ParamUtils.convert(paramsMap)); mapreduceParameters.setOthers(others); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java index 931adbf509..efa03545e9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class MapReduceTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml index 065d99386f..6c5063ee28 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-openmldb jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java index 27a1e349d3..592a5dad32 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class OpenmldbTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java index b4698c9461..32e3eaa04d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class OpenmldbTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new OpenmldbTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java index a8a70ce0a5..9345909ac9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java @@ -41,7 +41,8 @@ public class PigeonConfig { } private PigeonConfig() { - ResourceBundle bundle = ResourceBundle.getBundle(PigeonConfig.class.getPackage().getName().replace(".", "/") + "/config"); + ResourceBundle bundle = + ResourceBundle.getBundle(PigeonConfig.class.getPackage().getName().replace(".", "/") + "/config"); this.jobTriggerUrl = bundle.getString("job.trigger.url"); this.jobStatusUrl = bundle.getString("job.status.url"); this.jobTriggerPostBody = bundle.getString("job.trigger.post.body"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java index 72fff0277c..998544124c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class PigeonTaskChannel implements TaskChannel { + private static final Logger logger = LoggerFactory.getLogger(PigeonTaskChannel.class); @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java index e95f1330bb..a7ce316921 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java @@ -43,10 +43,10 @@ public class PigeonTaskChannelFactory implements TaskChannelFactory { @Override public List getParams() { - InputParam webHookParam - = InputParam.newBuilder(PigeonParamsConstants.NAME_TARGET_JOB_NAME, PigeonParamsConstants.TARGET_JOB_NAME) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + InputParam webHookParam = + InputParam.newBuilder(PigeonParamsConstants.NAME_TARGET_JOB_NAME, PigeonParamsConstants.TARGET_JOB_NAME) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); return Arrays.asList(webHookParam); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml index 586c47e3ec..2f2e67397a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-procedure jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java index b3a15a539e..bdb3ff4a27 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java @@ -135,9 +135,11 @@ public class ProcedureParameters extends AbstractParameters { } public ProcedureTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) { - DataSourceParameters dataSourceParameters = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource); + DataSourceParameters dataSourceParameters = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource); ProcedureTaskExecutionContext procedureTaskExecutionContext = new ProcedureTaskExecutionContext(); - procedureTaskExecutionContext.setConnectionParams(Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null); + procedureTaskExecutionContext.setConnectionParams( + Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null); return procedureTaskExecutionContext; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml index be3fc479e2..7145437bc7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-python jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java index 68e6c23e4b..d9fb27870d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters import java.util.List; public class PythonParameters extends AbstractParameters { + /** * origin python script */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java index 56dab9b2f2..d4b0b0b5f4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class PythonTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java index 00c44ee310..ae84f50bcd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.plugin.task.python;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; @@ -25,6 +25,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class PythonTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new PythonTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml index 0a0107d0e3..128c6f2199 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-pytorch jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java index d65e5e1150..bd11bb3546 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java @@ -26,16 +26,18 @@ import java.io.File; import java.nio.file.Paths; import java.util.regex.Pattern; +import lombok.Data; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import lombok.Data; - @Data public class GitProjectManager { + public static final String GIT_PATH_LOCAL = "GIT_PROJECT"; private static final Pattern GIT_CHECK_PATTERN = Pattern.compile("^(git@|https?://)"); - protected final Logger logger = LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); + protected final Logger logger = + LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); private String path; private String baseDir = "."; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java index cbac9b548e..7fbc669ae6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java @@ -54,7 +54,6 @@ public class PytorchParameters extends AbstractParameters { return true; } - public String getRequirementPath() { return getPossiblePath(requirements); } @@ -69,7 +68,6 @@ public class PytorchParameters extends AbstractParameters { return command; } - public String getScriptPath() { return getPossiblePath(script); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java index c362097e38..92aed842f6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java @@ -46,8 +46,8 @@ public class PytorchTask extends AbstractTask { this.taskExecutionContext = taskExecutionContext; this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, - taskExecutionContext, - logger); + taskExecutionContext, + logger); } @Override @@ -89,7 +89,6 @@ public class PytorchTask extends AbstractTask { } - public String buildPythonExecuteCommand() throws Exception { List args = new ArrayList<>(); @@ -112,7 +111,8 @@ public class PytorchTask extends AbstractTask { String scriptParams = pytorchParameters.getScriptParams(); if (scriptParams != null && !scriptParams.isEmpty()) { - args.add(String.format("%s %s %s", getPythonCommand(), pytorchParameters.getScriptPath(), pytorchParameters.getScriptParams())); + args.add(String.format("%s %s %s", getPythonCommand(), pytorchParameters.getScriptPath(), + pytorchParameters.getScriptParams())); } else { args.add(String.format("%s %s", getPythonCommand(), pytorchParameters.getScriptPath())); @@ -133,10 +133,8 @@ public class PytorchTask extends AbstractTask { } - @Override public AbstractParameters getParameters() { return pytorchParameters; } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java index be7e3d6545..bf1bda8b2d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java @@ -28,6 +28,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class PytorchTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new PytorchTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java index 143abbb644..73339d8da3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.sagemaker; public class SagemakerConstants { + public static final int CHECK_PIPELINE_EXECUTION_STATUS_INTERVAL = 5000; public static final int PIPELINE_MAX_RESULTS = 100; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java index 0c4afec462..ccad195e0e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java @@ -28,6 +28,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class SagemakerTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new SagemakerTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml index 5a0da75c7d..5816353168 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-seatunnel jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java index 28fb7685ba..0198877354 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.seatunnel; public enum DeployModeEnum { + cluster("cluster"), client("client"), local("client"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java index 38591ba8a4..6fa19fc0c6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java @@ -34,8 +34,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, visible = true, property = "engine") @JsonSubTypes({ - @JsonSubTypes.Type(value = SeatunnelFlinkParameters.class, name = "FLINK"), - @JsonSubTypes.Type(value = SeatunnelSparkParameters.class, name = "SPARK") + @JsonSubTypes.Type(value = SeatunnelFlinkParameters.class, name = "FLINK"), + @JsonSubTypes.Type(value = SeatunnelSparkParameters.class, name = "SPARK") }) public class SeatunnelParameters extends AbstractParameters { @@ -86,7 +86,8 @@ public class SeatunnelParameters extends AbstractParameters { public boolean checkParameters() { return Objects.nonNull(engine) && ((BooleanUtils.isTrue(useCustom) && StringUtils.isNotBlank(rawScript)) - || (BooleanUtils.isFalse(useCustom) && CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1)); + || (BooleanUtils.isFalse(useCustom) && CollectionUtils.isNotEmpty(resourceList) + && resourceList.size() == 1)); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java index b91c4f1228..b4dbc64902 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java @@ -35,7 +35,8 @@ public class SeatunnelTaskChannel implements TaskChannel { @Override public SeatunnelTask createTask(TaskExecutionContext taskRequest) { - SeatunnelParameters seatunnelParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), SeatunnelParameters.class); + SeatunnelParameters seatunnelParameters = + JSONUtils.parseObject(taskRequest.getTaskParams(), SeatunnelParameters.class); if (EngineEnum.FLINK == seatunnelParameters.getEngine()) { return new SeatunnelFlinkTask(taskRequest); } else if (EngineEnum.SPARK == seatunnelParameters.getEngine()) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java index 3f0b0eee40..0527bdaad2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java @@ -26,6 +26,7 @@ public class SeatunnelFlinkParameters extends SeatunnelParameters { private String others; public static enum RunModeEnum { + RUN("--run-mode run"), RUN_APPLICATION("--run-mode run-application"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java index efaa135334..07190375f9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java @@ -35,7 +35,8 @@ public class SeatunnelFlinkTask extends SeatunnelTask { @Override public void init() { - seatunnelParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelFlinkParameters.class); + seatunnelParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelFlinkParameters.class); setSeatunnelParameters(seatunnelParameters); super.init(); } @@ -43,7 +44,9 @@ public class SeatunnelFlinkTask extends SeatunnelTask { @Override public List buildOptions() throws Exception { List args = super.buildOptions(); - args.add(Objects.isNull(seatunnelParameters.getRunMode()) ? SeatunnelFlinkParameters.RunModeEnum.RUN.getCommand() : seatunnelParameters.getRunMode().getCommand()); + args.add( + Objects.isNull(seatunnelParameters.getRunMode()) ? SeatunnelFlinkParameters.RunModeEnum.RUN.getCommand() + : seatunnelParameters.getRunMode().getCommand()); if (StringUtils.isNotBlank(seatunnelParameters.getOthers())) { args.add(seatunnelParameters.getOthers()); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java index e17675a845..79c543a74d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java @@ -43,6 +43,7 @@ public class SeatunnelSparkParameters extends SeatunnelParameters { } public static enum MasterTypeEnum { + YARN("yarn"), LOCAL("local"), SPARK("spark://"), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java index 9fcb5a424a..e6f4b4417a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.seatunnel.spark; import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.DEPLOY_MODE_OPTIONS; import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.MASTER_OPTIONS; -import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.QUEUE_OPTIONS; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.seatunnel.DeployModeEnum; @@ -27,8 +26,6 @@ import org.apache.dolphinscheduler.plugin.task.seatunnel.SeatunnelTask; import org.apache.dolphinscheduler.plugin.task.seatunnel.spark.SeatunnelSparkParameters.MasterTypeEnum; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.commons.lang3.StringUtils; - import java.util.List; public class SeatunnelSparkTask extends SeatunnelTask { @@ -45,7 +42,8 @@ public class SeatunnelSparkTask extends SeatunnelTask { @Override public void init() { - seatunnelParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelSparkParameters.class); + seatunnelParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelSparkParameters.class); setSeatunnelParameters(seatunnelParameters); super.init(); } @@ -56,7 +54,8 @@ public class SeatunnelSparkTask extends SeatunnelTask { args.add(DEPLOY_MODE_OPTIONS); args.add(seatunnelParameters.getDeployMode().getCommand()); - MasterTypeEnum master = DeployModeEnum.local == seatunnelParameters.getDeployMode() ? MasterTypeEnum.LOCAL : seatunnelParameters.getMaster(); + MasterTypeEnum master = DeployModeEnum.local == seatunnelParameters.getDeployMode() ? MasterTypeEnum.LOCAL + : seatunnelParameters.getMaster(); args.add(MASTER_OPTIONS); args.add(master.getCommand()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml index e7ddd6dd19..73fc5066f0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-shell jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java index 7c7c4b2497..6e83c5c268 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.shell; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; @@ -32,6 +32,8 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.commons.lang3.SystemUtils; + import java.io.File; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -43,9 +45,6 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.Map; import java.util.Set; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; - /** * shell task */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java index c960a0d5c1..801826ffd1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java @@ -32,6 +32,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class ShellTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new ShellTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml index 45c2923c28..6d0d6d79b1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-spark jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java index 0d307bde55..92db56ad28 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class SparkTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "SPARK"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml index 4896aecc4f..501c6be998 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-sql jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java index 5f2741d952..3b61a53626 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java @@ -25,6 +25,7 @@ import java.util.Map; * Used to contains both prepared sql string and its to-be-bind parameters */ public class SqlBinds { + private final String sql; private final Map paramsMap; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 2322fc27ff..ed6527d4ce 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -20,7 +20,12 @@ package org.apache.dolphinscheduler.plugin.task.sql; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; -import org.apache.dolphinscheduler.plugin.task.api.*; +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.SQLTaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; @@ -82,7 +87,8 @@ public class SqlTask extends AbstractTask { * create function format * include replace here which can be compatible with more cases, for example a long-running Spark session in Kyuubi will keep its own temp functions instead of destroying them right away */ - private static final String CREATE_OR_REPLACE_FUNCTION_FORMAT = "create or replace temporary function {0} as ''{1}''"; + private static final String CREATE_OR_REPLACE_FUNCTION_FORMAT = + "create or replace temporary function {0} as ''{1}''"; /** * default query sql limit @@ -113,7 +119,8 @@ public class SqlTask extends AbstractTask { throw new RuntimeException("sql task params is not valid"); } - sqlTaskExecutionContext = sqlParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + sqlTaskExecutionContext = + sqlParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); } @Override @@ -124,7 +131,8 @@ public class SqlTask extends AbstractTask { @Override public void handle(TaskCallBack taskCallBack) throws TaskException { logger.info("Full sql parameters: {}", sqlParameters); - logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {},varPool : {} ,query max result limit {}", + logger.info( + "sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {},varPool : {} ,query max result limit {}", sqlParameters.getType(), sqlParameters.getDatasource(), sqlParameters.getSql(), @@ -221,7 +229,8 @@ public class SqlTask extends AbstractTask { try { // create connection - connection = DataSourceClientProvider.getInstance().getConnection(DbType.valueOf(sqlParameters.getType()), baseConnectionParam); + connection = DataSourceClientProvider.getInstance().getConnection(DbType.valueOf(sqlParameters.getType()), + baseConnectionParam); // create temp function if (CollectionUtils.isNotEmpty(createFuncs)) { createTempFunction(connection, createFuncs); @@ -241,7 +250,7 @@ public class SqlTask extends AbstractTask { String updateResult = executeUpdate(connection, mainStatementsBinds, "main"); result = setNonQuerySqlReturn(updateResult, sqlParameters.getLocalParams()); } - //deal out params + // deal out params sqlParameters.dealOutParam(result); // post execute @@ -297,7 +306,8 @@ public class SqlTask extends AbstractTask { rowCount++; } - int displayRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() : TaskConstants.DEFAULT_DISPLAY_ROWS; + int displayRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() + : TaskConstants.DEFAULT_DISPLAY_ROWS; displayRows = Math.min(displayRows, rowCount); logger.info("display sql result {} rows as follows:", displayRows); for (int i = 0; i < displayRows; i++) { @@ -306,8 +316,8 @@ public class SqlTask extends AbstractTask { } } - String result = resultJSONArray.isEmpty() ? - JSONUtils.toJsonString(generateEmptyRow(resultSet)) : JSONUtils.toJsonString(resultJSONArray); + String result = resultJSONArray.isEmpty() ? JSONUtils.toJsonString(generateEmptyRow(resultSet)) + : JSONUtils.toJsonString(resultJSONArray); if (sqlParameters.getSendEmail() == null || sqlParameters.getSendEmail()) { sendAttachment(sqlParameters.getGroupId(), StringUtils.isNotEmpty(sqlParameters.getTitle()) @@ -338,7 +348,6 @@ public class SqlTask extends AbstractTask { return resultJSONArray; } - /** * send alert as an attachment * @@ -362,12 +371,14 @@ public class SqlTask extends AbstractTask { } } - private String executeUpdate(Connection connection, List statementsBinds, String handlerType) throws Exception { + private String executeUpdate(Connection connection, List statementsBinds, + String handlerType) throws Exception { int result = 0; for (SqlBinds sqlBind : statementsBinds) { try (PreparedStatement statement = prepareStatementAndBind(connection, sqlBind)) { result = statement.executeUpdate(); - logger.info("{} statement execute update result: {}, for sql: {}", handlerType, result, sqlBind.getSql()); + logger.info("{} statement execute update result: {}, for sql: {}", handlerType, result, + sqlBind.getSql()); } } return String.valueOf(result); @@ -428,7 +439,8 @@ public class SqlTask extends AbstractTask { ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); } } - logger.info("prepare statement replace sql : {}, sql parameters : {}", sqlBinds.getSql(), sqlBinds.getParamsMap()); + logger.info("prepare statement replace sql : {}, sql parameters : {}", sqlBinds.getSql(), + sqlBinds.getParamsMap()); return stmt; } catch (Exception exception) { throw new TaskException("SQL task prepareStatementAndBind error", exception); @@ -444,14 +456,15 @@ public class SqlTask extends AbstractTask { * @param sqlParamsMap sql params map */ private void printReplacedSql(String content, String formatSql, String rgex, Map sqlParamsMap) { - //parameter print style + // parameter print style logger.info("after replace sql , preparing : {}", formatSql); StringBuilder logPrint = new StringBuilder("replaced sql , parameters:"); if (sqlParamsMap == null) { logger.info("printReplacedSql: sqlParamsMap is null."); } else { for (int i = 1; i <= sqlParamsMap.size(); i++) { - logPrint.append(sqlParamsMap.get(i).getValue()).append("(").append(sqlParamsMap.get(i).getType()).append(")"); + logPrint.append(sqlParamsMap.get(i).getValue()).append("(").append(sqlParamsMap.get(i).getType()) + .append(")"); } } logger.info("Sql Params are {}", logPrint); @@ -482,12 +495,13 @@ public class SqlTask extends AbstractTask { sqlParameters.setTitle(title); } - //new - //replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job - sql = ParameterUtils.replaceScheduleTime(sql, DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime())); + // new + // replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job + sql = ParameterUtils.replaceScheduleTime(sql, + DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime())); // special characters need to be escaped, ${} needs to be escaped - setSqlParamsMap(sql, rgex, sqlParamsMap, paramsMap,taskExecutionContext.getTaskInstanceId()); - //Replace the original value in sql ！{...} ，Does not participate in precompilation + setSqlParamsMap(sql, rgex, sqlParamsMap, paramsMap, taskExecutionContext.getTaskInstanceId()); + // Replace the original value in sql ！{...} ，Does not participate in precompilation String rgexo = "['\"]*\\!\\{(.*?)\\}['\"]*"; sql = replaceOriginalValue(sql, rgexo, paramsMap); // replace the ${} of the SQL statement with the Placeholder @@ -543,7 +557,8 @@ public class SqlTask extends AbstractTask { */ private List buildTempFuncSql(List udfFuncParameters) { return udfFuncParameters.stream().map(value -> MessageFormat - .format(CREATE_OR_REPLACE_FUNCTION_FORMAT, value.getFuncName(), value.getClassName())).collect(Collectors.toList()); + .format(CREATE_OR_REPLACE_FUNCTION_FORMAT, value.getFuncName(), value.getClassName())) + .collect(Collectors.toList()); } /** @@ -557,7 +572,8 @@ public class SqlTask extends AbstractTask { String prefixPath = defaultFS.startsWith("file://") ? "file://" : defaultFS; String uploadPath = CommonUtils.getHdfsUdfDir(value.getTenantCode()); String resourceFullName = value.getResourceName(); - resourceFullName = resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s", resourceFullName); + resourceFullName = + resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s", resourceFullName); return String.format("add jar %s%s%s", prefixPath, uploadPath, resourceFullName); }).collect(Collectors.toList()); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java index 390ef863fb..4d9b64076c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceP import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class SqlTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java index 7389dcad0b..909741edfc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class SqlTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "SQL"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml index d285977a44..c44508e774 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-sqoop jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java index 002f06bf9c..f4861eb4fb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java @@ -22,7 +22,7 @@ public final class SqoopConstants { private SqoopConstants() { } - //sqoop general param + // sqoop general param public static final String SQOOP = "sqoop"; public static final String SQOOP_MR_JOB_NAME = "mapred.job.name"; public static final String SQOOP_PARALLELISM = "-m"; @@ -32,7 +32,7 @@ public final class SqoopConstants { public static final String SPLIT_BY = "--split-by"; - //sqoop db + // sqoop db public static final String DB_CONNECT = "--connect"; public static final String DB_USERNAME = "--username"; public static final String DB_PWD = "--password"; @@ -45,19 +45,18 @@ public final class SqoopConstants { public static final String MAP_COLUMN_HIVE = "--map-column-hive"; public static final String MAP_COLUMN_JAVA = "--map-column-java"; - - //sqoop hive source + // sqoop hive source public static final String HCATALOG_DATABASE = "--hcatalog-database"; public static final String HCATALOG_TABLE = "--hcatalog-table"; public static final String HCATALOG_PARTITION_KEYS = "--hcatalog-partition-keys"; public static final String HCATALOG_PARTITION_VALUES = "--hcatalog-partition-values"; - //sqoop hdfs + // sqoop hdfs public static final String HDFS_EXPORT_DIR = "--export-dir"; public static final String TARGET_DIR = "--target-dir"; public static final String COMPRESSION_CODEC = "--compression-codec"; - //sqoop hive + // sqoop hive public static final String HIVE_IMPORT = "--hive-import"; public static final String HIVE_DATABASE = "--hive-database"; public static final String HIVE_TABLE = "--hive-table"; @@ -69,9 +68,8 @@ public final class SqoopConstants { public static final String HIVE_PARTITION_KEY = "--hive-partition-key"; public static final String HIVE_PARTITION_VALUE = "--hive-partition-value"; - //sqoop update model + // sqoop update model public static final String UPDATE_KEY = "--update-key"; public static final String UPDATE_MODE = "--update-mode"; - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java index 407390ef21..1b5dfc6074 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.sqoop; -public enum SqoopJobType { +public enum SqoopJobType { CUSTOM(0, "CUSTOM"), TEMPLATE(1, "TEMPLATE"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java index fe4c428576..f61256797d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java @@ -23,12 +23,10 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.plugin.task.sqoop.generator.SqoopJobGenerator; import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import java.util.HashMap; import java.util.Map; /** @@ -57,8 +55,8 @@ public class SqoopTask extends AbstractYarnTask { public void init() { logger.info("sqoop task params {}", taskExecutionContext.getTaskParams()); sqoopParameters = - JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqoopParameters.class); - //check sqoop task params + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqoopParameters.class); + // check sqoop task params if (null == sqoopParameters) { throw new IllegalArgumentException("Sqoop Task params is null"); } @@ -67,12 +65,13 @@ public class SqoopTask extends AbstractYarnTask { throw new IllegalArgumentException("Sqoop Task params check fail"); } - sqoopTaskExecutionContext = sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + sqoopTaskExecutionContext = + sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); } @Override protected String buildCommand() { - //get sqoop scripts + // get sqoop scripts SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java index 5cae89e7a4..0426906361 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java @@ -44,44 +44,44 @@ public class CommonGenerator { StringBuilder commonSb = new StringBuilder(); try { - //sqoop task model + // sqoop task model commonSb.append(SqoopConstants.SQOOP) - .append(SPACE) - .append(sqoopParameters.getModelType()); + .append(SPACE) + .append(sqoopParameters.getModelType()); - //sqoop map-reduce job name + // sqoop map-reduce job name commonSb.append(SPACE).append(D).append(SPACE) - .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME, - EQUAL_SIGN, sqoopParameters.getJobName())); + .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME, + EQUAL_SIGN, sqoopParameters.getJobName())); - //hadoop custom param + // hadoop custom param List hadoopCustomParams = sqoopParameters.getHadoopCustomParams(); if (CollectionUtils.isNotEmpty(hadoopCustomParams)) { for (Property hadoopCustomParam : hadoopCustomParams) { String hadoopCustomParamStr = String.format("%s%s%s", hadoopCustomParam.getProp(), - EQUAL_SIGN, hadoopCustomParam.getValue()); + EQUAL_SIGN, hadoopCustomParam.getValue()); commonSb.append(SPACE).append(D) - .append(SPACE).append(hadoopCustomParamStr); + .append(SPACE).append(hadoopCustomParamStr); } } - //sqoop custom params + // sqoop custom params List sqoopAdvancedParams = sqoopParameters.getSqoopAdvancedParams(); if (CollectionUtils.isNotEmpty(sqoopAdvancedParams)) { for (Property sqoopAdvancedParam : sqoopAdvancedParams) { commonSb.append(SPACE).append(sqoopAdvancedParam.getProp()) - .append(SPACE).append(sqoopAdvancedParam.getValue()); + .append(SPACE).append(sqoopAdvancedParam.getValue()); } } - //sqoop parallelism + // sqoop parallelism if (sqoopParameters.getConcurrency() > 0) { commonSb.append(SPACE).append(SqoopConstants.SQOOP_PARALLELISM) - .append(SPACE).append(sqoopParameters.getConcurrency()); + .append(SPACE).append(sqoopParameters.getConcurrency()); if (sqoopParameters.getConcurrency() > 1) { commonSb.append(SPACE).append(SqoopConstants.SPLIT_BY) - .append(SPACE).append(sqoopParameters.getSplitBy()); + .append(SPACE).append(sqoopParameters.getSplitBy()); } } } catch (Exception e) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java index 67bd60880a..48d152f70d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java @@ -65,7 +65,8 @@ public class SqoopJobGenerator { * @param sqoopTaskExecutionContext * @return sqoop scripts */ - public String generateSqoopJob(SqoopParameters sqoopParameters, SqoopTaskExecutionContext sqoopTaskExecutionContext) { + public String generateSqoopJob(SqoopParameters sqoopParameters, + SqoopTaskExecutionContext sqoopTaskExecutionContext) { String sqoopScripts = ""; @@ -76,8 +77,8 @@ public class SqoopJobGenerator { } sqoopScripts = String.format("%s%s%s", commonGenerator.generate(sqoopParameters), - sourceGenerator.generate(sqoopParameters, sqoopTaskExecutionContext), - targetGenerator.generate(sqoopParameters, sqoopTaskExecutionContext)); + sourceGenerator.generate(sqoopParameters, sqoopTaskExecutionContext), + targetGenerator.generate(sqoopParameters, sqoopTaskExecutionContext)); } else if (SqoopJobType.CUSTOM.getDescp().equals(sqoopParameters.getJobType())) { sqoopScripts = sqoopParameters.getCustomShell().replaceAll("\\r\\n", "\n"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java index 652aff530e..d21ab912be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java @@ -43,13 +43,13 @@ public class HdfsSourceGenerator implements ISourceGenerator { StringBuilder hdfsSourceSb = new StringBuilder(); try { - SourceHdfsParameter sourceHdfsParameter - = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class); + SourceHdfsParameter sourceHdfsParameter = + JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class); if (null != sourceHdfsParameter) { if (StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())) { hdfsSourceSb.append(SPACE).append(HDFS_EXPORT_DIR) - .append(SPACE).append(sourceHdfsParameter.getExportDir()); + .append(SPACE).append(sourceHdfsParameter.getExportDir()); } else { throw new IllegalArgumentException("Sqoop hdfs export dir is null"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java index de1c9597b2..f9a2881b5f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java @@ -46,26 +46,26 @@ public class HiveSourceGenerator implements ISourceGenerator { StringBuilder hiveSourceSb = new StringBuilder(); try { - SourceHiveParameter sourceHiveParameter - = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class); + SourceHiveParameter sourceHiveParameter = + JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class); if (null != sourceHiveParameter) { if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())) { hiveSourceSb.append(SPACE).append(HCATALOG_DATABASE) - .append(SPACE).append(sourceHiveParameter.getHiveDatabase()); + .append(SPACE).append(sourceHiveParameter.getHiveDatabase()); } if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())) { hiveSourceSb.append(SPACE).append(HCATALOG_TABLE) - .append(SPACE).append(sourceHiveParameter.getHiveTable()); + .append(SPACE).append(sourceHiveParameter.getHiveTable()); } if (StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey()) - && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) { + && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) { hiveSourceSb.append(SPACE).append(HCATALOG_PARTITION_KEYS) - .append(SPACE).append(sourceHiveParameter.getHivePartitionKey()) - .append(SPACE).append(HCATALOG_PARTITION_VALUES) - .append(SPACE).append(sourceHiveParameter.getHivePartitionValue()); + .append(SPACE).append(sourceHiveParameter.getHivePartitionKey()) + .append(SPACE).append(HCATALOG_PARTITION_VALUES) + .append(SPACE).append(sourceHiveParameter.getHivePartitionValue()); } } } catch (Exception e) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java index 8e8ab4cdc9..e434f7360b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java @@ -64,7 +64,8 @@ public class MySQLSourceGenerator implements ISourceGenerator { StringBuilder mysqlSourceSb = new StringBuilder(); try { - SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class); + SourceMysqlParameter sourceMysqlParameter = + JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class); if (null != sourceMysqlParameter) { BaseConnectionParam baseDataSource = (BaseConnectionParam) DataSourceUtils.buildConnectionParams( @@ -76,29 +77,29 @@ public class MySQLSourceGenerator implements ISourceGenerator { mysqlSourceSb.append(SPACE).append(DB_CONNECT) .append(SPACE).append(DOUBLE_QUOTES) .append(DataSourceUtils.getJdbcUrl(DbType.MYSQL, baseDataSource)).append(DOUBLE_QUOTES) - .append(SPACE).append(DB_USERNAME) - .append(SPACE).append(baseDataSource.getUser()) - .append(SPACE).append(DB_PWD) - .append(SPACE).append(DOUBLE_QUOTES) + .append(SPACE).append(DB_USERNAME) + .append(SPACE).append(baseDataSource.getUser()) + .append(SPACE).append(DB_PWD) + .append(SPACE).append(DOUBLE_QUOTES) .append(decodePassword(baseDataSource.getPassword())).append(DOUBLE_QUOTES); - //sqoop table & sql query + // sqoop table & sql query if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.FORM.getCode()) { if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())) { mysqlSourceSb.append(SPACE).append(TABLE) - .append(SPACE).append(sourceMysqlParameter.getSrcTable()); + .append(SPACE).append(sourceMysqlParameter.getSrcTable()); } if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())) { mysqlSourceSb.append(SPACE).append(COLUMNS) - .append(SPACE).append(sourceMysqlParameter.getSrcColumns()); + .append(SPACE).append(sourceMysqlParameter.getSrcColumns()); } } else if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.SQL.getCode() - && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) { + && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) { String srcQuery = sourceMysqlParameter.getSrcQuerySql(); mysqlSourceSb.append(SPACE).append(QUERY) - .append(SPACE).append(DOUBLE_QUOTES).append(srcQuery); + .append(SPACE).append(DOUBLE_QUOTES).append(srcQuery); if (srcQuery.toLowerCase().contains(QUERY_WHERE)) { mysqlSourceSb.append(SPACE).append(QUERY_CONDITION).append(DOUBLE_QUOTES); @@ -107,7 +108,7 @@ public class MySQLSourceGenerator implements ISourceGenerator { } } - //sqoop hive map column + // sqoop hive map column List mapColumnHive = sourceMysqlParameter.getMapColumnHive(); if (null != mapColumnHive && !mapColumnHive.isEmpty()) { @@ -118,11 +119,11 @@ public class MySQLSourceGenerator implements ISourceGenerator { if (StringUtils.isNotEmpty(columnMap.toString())) { mysqlSourceSb.append(SPACE).append(MAP_COLUMN_HIVE) - .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); + .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); } } - //sqoop map column java + // sqoop map column java List mapColumnJava = sourceMysqlParameter.getMapColumnJava(); if (null != mapColumnJava && !mapColumnJava.isEmpty()) { @@ -133,7 +134,7 @@ public class MySQLSourceGenerator implements ISourceGenerator { if (StringUtils.isNotEmpty(columnMap.toString())) { mysqlSourceSb.append(SPACE).append(MAP_COLUMN_JAVA) - .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); + .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java index 4a619ca1f1..d2c1b2ed40 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java @@ -50,18 +50,18 @@ public class HdfsTargetGenerator implements ITargetGenerator { try { TargetHdfsParameter targetHdfsParameter = - JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class); + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class); if (null != targetHdfsParameter) { if (StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())) { hdfsTargetSb.append(SPACE).append(TARGET_DIR) - .append(SPACE).append(targetHdfsParameter.getTargetPath()); + .append(SPACE).append(targetHdfsParameter.getTargetPath()); } if (StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())) { hdfsTargetSb.append(SPACE).append(COMPRESSION_CODEC) - .append(SPACE).append(targetHdfsParameter.getCompressionCodec()); + .append(SPACE).append(targetHdfsParameter.getCompressionCodec()); } if (StringUtils.isNotEmpty(targetHdfsParameter.getFileType())) { @@ -74,12 +74,14 @@ public class HdfsTargetGenerator implements ITargetGenerator { if (StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())) { hdfsTargetSb.append(SPACE).append(FIELDS_TERMINATED_BY) - .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()).append(SINGLE_QUOTES); + .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()) + .append(SINGLE_QUOTES); } if (StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())) { hdfsTargetSb.append(SPACE).append(LINES_TERMINATED_BY) - .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()).append(SINGLE_QUOTES); + .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()) + .append(SINGLE_QUOTES); } hdfsTargetSb.append(SPACE).append(FIELD_NULL_PLACEHOLDER); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java index 0fba0fed40..c7b5071276 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java @@ -54,16 +54,16 @@ public class HiveTargetGenerator implements ITargetGenerator { try { TargetHiveParameter targetHiveParameter = - JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class); + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class); if (null != targetHiveParameter) { hiveTargetSb.append(SPACE).append(HIVE_IMPORT); if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase()) - && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) { + && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) { hiveTargetSb.append(SPACE).append(HIVE_DATABASE) - .append(SPACE).append(targetHiveParameter.getHiveDatabase()) - .append(SPACE).append(HIVE_TABLE) - .append(SPACE).append(targetHiveParameter.getHiveTable()); + .append(SPACE).append(targetHiveParameter.getHiveDatabase()) + .append(SPACE).append(HIVE_TABLE) + .append(SPACE).append(targetHiveParameter.getHiveTable()); } if (targetHiveParameter.isCreateHiveTable()) { @@ -76,20 +76,20 @@ public class HiveTargetGenerator implements ITargetGenerator { if (targetHiveParameter.isHiveOverWrite()) { hiveTargetSb.append(SPACE).append(HIVE_OVERWRITE) - .append(SPACE).append(DELETE_TARGET_DIR); + .append(SPACE).append(DELETE_TARGET_DIR); } if (StringUtils.isNotEmpty(targetHiveParameter.getReplaceDelimiter())) { hiveTargetSb.append(SPACE).append(HIVE_DELIMS_REPLACEMENT) - .append(SPACE).append(targetHiveParameter.getReplaceDelimiter()); + .append(SPACE).append(targetHiveParameter.getReplaceDelimiter()); } if (StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionKey()) - && StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())) { + && StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())) { hiveTargetSb.append(SPACE).append(HIVE_PARTITION_KEY) - .append(SPACE).append(targetHiveParameter.getHivePartitionKey()) - .append(SPACE).append(HIVE_PARTITION_VALUE) - .append(SPACE).append(targetHiveParameter.getHivePartitionValue()); + .append(SPACE).append(targetHiveParameter.getHivePartitionKey()) + .append(SPACE).append(HIVE_PARTITION_VALUE) + .append(SPACE).append(targetHiveParameter.getHivePartitionValue()); } if (StringUtils.isNotEmpty(targetHiveParameter.getHiveTargetDir())) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java index e8f7e85a5f..94f912d97e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java @@ -58,7 +58,7 @@ public class MySQLTargetGenerator implements ITargetGenerator { try { TargetMysqlParameter targetMysqlParameter = - JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class); + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class); if (null != targetMysqlParameter && targetMysqlParameter.getTargetDatasource() != 0) { @@ -77,12 +77,12 @@ public class MySQLTargetGenerator implements ITargetGenerator { .append(SPACE).append(DB_PWD) .append(SPACE).append(DOUBLE_QUOTES) .append(decodePassword(baseDataSource.getPassword())).append(DOUBLE_QUOTES) - .append(SPACE).append(TABLE) - .append(SPACE).append(targetMysqlParameter.getTargetTable()); + .append(SPACE).append(TABLE) + .append(SPACE).append(targetMysqlParameter.getTargetTable()); if (StringUtils.isNotEmpty(targetMysqlParameter.getTargetColumns())) { mysqlTargetSb.append(SPACE).append(COLUMNS) - .append(SPACE).append(targetMysqlParameter.getTargetColumns()); + .append(SPACE).append(targetMysqlParameter.getTargetColumns()); } if (StringUtils.isNotEmpty(targetMysqlParameter.getFieldsTerminated())) { @@ -91,7 +91,8 @@ public class MySQLTargetGenerator implements ITargetGenerator { mysqlTargetSb.append(SPACE).append(targetMysqlParameter.getFieldsTerminated()); } else { - mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES).append(targetMysqlParameter.getFieldsTerminated()).append(SINGLE_QUOTES); + mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES) + .append(targetMysqlParameter.getFieldsTerminated()).append(SINGLE_QUOTES); } } @@ -100,17 +101,18 @@ public class MySQLTargetGenerator implements ITargetGenerator { if (targetMysqlParameter.getLinesTerminated().contains(SINGLE_QUOTES)) { mysqlTargetSb.append(SPACE).append(targetMysqlParameter.getLinesTerminated()); } else { - mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES).append(targetMysqlParameter.getLinesTerminated()).append(SINGLE_QUOTES); + mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES) + .append(targetMysqlParameter.getLinesTerminated()).append(SINGLE_QUOTES); } } if (targetMysqlParameter.getIsUpdate() - && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) - && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())) { + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())) { mysqlTargetSb.append(SPACE).append(UPDATE_KEY) - .append(SPACE).append(targetMysqlParameter.getTargetUpdateKey()) - .append(SPACE).append(UPDATE_MODE) - .append(SPACE).append(targetMysqlParameter.getTargetUpdateMode()); + .append(SPACE).append(targetMysqlParameter.getTargetUpdateKey()) + .append(SPACE).append(UPDATE_MODE) + .append(SPACE).append(targetMysqlParameter.getTargetUpdateMode()); } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java index 3d88b288f0..d5d88506b5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java @@ -200,16 +200,16 @@ public class SqoopParameters extends AbstractParameters { if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) { sqoopParamsCheck = StringUtils.isEmpty(customShell) - && StringUtils.isNotEmpty(modelType) - && StringUtils.isNotEmpty(jobName) - && concurrency != 0 - && StringUtils.isNotEmpty(sourceType) - && StringUtils.isNotEmpty(targetType) - && StringUtils.isNotEmpty(sourceParams) - && StringUtils.isNotEmpty(targetParams); + && StringUtils.isNotEmpty(modelType) + && StringUtils.isNotEmpty(jobName) + && concurrency != 0 + && StringUtils.isNotEmpty(sourceType) + && StringUtils.isNotEmpty(targetType) + && StringUtils.isNotEmpty(sourceParams) + && StringUtils.isNotEmpty(targetParams); } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) { sqoopParamsCheck = StringUtils.isNotEmpty(customShell) - && StringUtils.isEmpty(jobName); + && StringUtils.isEmpty(jobName); } return sqoopParamsCheck; @@ -222,12 +222,14 @@ public class SqoopParameters extends AbstractParameters { return resources; } - SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); + SourceMysqlParameter sourceMysqlParameter = + JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); if (sourceMysqlParameter.getSrcDatasource() != 0) { resources.put(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource()); } - TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); + TargetMysqlParameter targetMysqlParameter = + JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); if (targetMysqlParameter.getTargetDatasource() != 0) { resources.put(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource()); } @@ -242,11 +244,15 @@ public class SqoopParameters extends AbstractParameters { return sqoopTaskExecutionContext; } - SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); - TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); + SourceMysqlParameter sourceMysqlParameter = + JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); + TargetMysqlParameter targetMysqlParameter = + JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); - DataSourceParameters dataSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource()); - DataSourceParameters dataTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource()); + DataSourceParameters dataSource = (DataSourceParameters) parametersHelper + .getResourceParameters(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource()); + DataSourceParameters dataTarget = (DataSourceParameters) parametersHelper + .getResourceParameters(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource()); if (Objects.nonNull(dataSource)) { sqoopTaskExecutionContext.setDataSourceId(sourceMysqlParameter.getSrcDatasource()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java index 37388dbf1d..e1d10d1b7d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java @@ -31,12 +31,13 @@ import java.util.Map; public class EntityTestUtils { private static final Map OBJECT_MAP = new HashMap<>(); - + private static final String SKIP_METHOD = "getClass,notify,notifyAll,wait,equals,hashCode,clone"; static { OBJECT_MAP.put("java.lang.Long", 1L); - OBJECT_MAP.put("java.lang.String", "[{\"direct\":\"OUT\",\"prop\":\"percentage5\",\"type\":\"VARCHAR\",\"value\":\"qwe\"}]"); + OBJECT_MAP.put("java.lang.String", + "[{\"direct\":\"OUT\",\"prop\":\"percentage5\",\"type\":\"VARCHAR\",\"value\":\"qwe\"}]"); OBJECT_MAP.put("java.lang.Integer", 1); OBJECT_MAP.put("int", 1); OBJECT_MAP.put("long", 1L); @@ -45,9 +46,8 @@ public class EntityTestUtils { OBJECT_MAP.put("java.util.Map", new HashMap()); OBJECT_MAP.put("boolean", true); } - - public static void run(List classList) - throws IllegalAccessException, InvocationTargetException, InstantiationException { + + public static void run(List classList) throws IllegalAccessException, InvocationTargetException, InstantiationException { for (Class temp : classList) { Object tempInstance = new Object(); Constructor[] constructors = temp.getConstructors(); @@ -82,4 +82,4 @@ public class EntityTestUtils { } } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml index f723207cad..383e1d9423 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-subprocess jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java index 18f356b944..c440e265d3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java @@ -29,6 +29,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class SubProcessTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new SubProcessTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml index 41159491f7..a8a9892b59 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-switch jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java index d92fd45993..37287aec03 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java @@ -79,15 +79,16 @@ public class SwitchParameters extends AbstractParameters { @Override public boolean checkParameters() { - //default next node should not be null + // default next node should not be null boolean defaultNode = switchResult != null && switchResult.getNextNode() != null; if (!defaultNode) { return false; } - //validate conditions must have next node + // validate conditions must have next node List conditions = this.switchResult.getDependTaskList(); if (conditions != null && conditions.size() != 0) { - if (conditions.stream().anyMatch(e -> (StringUtils.isNotEmpty(e.getCondition()) && e.getNextNode() == null))) { + if (conditions.stream() + .anyMatch(e -> (StringUtils.isNotEmpty(e.getCondition()) && e.getNextNode() == null))) { return false; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java index da67918a61..f6bb53e719 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.switchtask; import java.util.List; public class SwitchResult { + private List dependTaskList; private Long nextNode; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java index fb41a0b677..4a25905d79 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.switchtask; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; @@ -25,8 +24,11 @@ import org.apache.dolphinscheduler.spi.params.base.PluginParams; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(TaskChannelFactory.class) public class SwitchTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new SwitchTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java index b95c93971f..9f45bb952f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -27,6 +24,10 @@ import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.Collections; import java.util.List; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + @Getter @Setter @ToString diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java index aefe54129c..006735875c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java @@ -27,6 +27,7 @@ import com.google.auto.service.AutoService; @AutoService(TaskChannelFactory.class) public class ZeppelinTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "ZEPPELIN"; diff --git a/dolphinscheduler-tools/pom.xml b/dolphinscheduler-tools/pom.xml index 4212bcf4bf..f82acdcdf8 100644 --- a/dolphinscheduler-tools/pom.xml +++ b/dolphinscheduler-tools/pom.xml @@ -18,25 +18,17 @@ ~ under the License. ~ --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-tools - - - org.apache.dolphinscheduler - dolphinscheduler-dao - - - @@ -49,6 +41,13 @@ + + + org.apache.dolphinscheduler + dolphinscheduler-dao + + + @@ -65,10 +64,10 @@ dolphinscheduler-tools - package single + package tools diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java index a588fabcfb..b56d103606 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java @@ -33,6 +33,7 @@ import org.springframework.stereotype.Service; @Service public class DolphinSchedulerManager { + private static final Logger logger = LoggerFactory.getLogger(DolphinSchedulerManager.class); private final UpgradeDao upgradeDao; @@ -40,11 +41,10 @@ public class DolphinSchedulerManager { public DolphinSchedulerManager(DataSource dataSource, List daos) throws Exception { final DbType type = getCurrentDbType(dataSource); upgradeDao = daos.stream() - .filter(it -> it.getDbType() == type) - .findFirst() - .orElseThrow(() -> new RuntimeException( - "Cannot find UpgradeDao implementation for db type: " + type - )); + .filter(it -> it.getDbType() == type) + .findFirst() + .orElseThrow(() -> new RuntimeException( + "Cannot find UpgradeDao implementation for db type: " + type)); } private DbType getCurrentDbType(DataSource dataSource) throws Exception { @@ -65,8 +65,8 @@ public class DolphinSchedulerManager { public boolean schemaIsInitialized() { // Determines whether the dolphinscheduler table structure has been init if (upgradeDao.isExistsTable("t_escheduler_version") - || upgradeDao.isExistsTable("t_ds_version") - || upgradeDao.isExistsTable("t_escheduler_queue")) { + || upgradeDao.isExistsTable("t_ds_version") + || upgradeDao.isExistsTable("t_escheduler_queue")) { logger.info("The database has been initialized. Skip the initialization step"); return true; } @@ -106,7 +106,7 @@ public class DolphinSchedulerManager { if (SchemaUtils.isAGreatVersion(schemaVersion, version)) { logger.info("upgrade DolphinScheduler metadata version from {} to {}", version, schemaVersion); logger.info("Begin upgrading DolphinScheduler's table structure"); - upgradeDao.upgradeDolphinScheduler(schemaDir); + upgradeDao.upgradeDolphinScheduler(schemaDir); if ("1.3.0".equals(schemaVersion)) { upgradeDao.upgradeDolphinSchedulerWorkerGroup(); } else if ("1.3.2".equals(schemaVersion)) { @@ -118,7 +118,8 @@ public class DolphinSchedulerManager { } } - if (SchemaUtils.isAGreatVersion("2.0.6", currentVersion) && SchemaUtils.isAGreatVersion(SchemaUtils.getSoftVersion(), currentVersion)) { + if (SchemaUtils.isAGreatVersion("2.0.6", currentVersion) + && SchemaUtils.isAGreatVersion(SchemaUtils.getSoftVersion(), currentVersion)) { upgradeDao.upgradeDolphinSchedulerResourceFileSize(); } } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java index 6ccd73decd..15b1cbb3b2 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Component; @SpringBootApplication public class InitDolphinScheduler { + public static void main(String[] args) { SpringApplication.run(InitDolphinScheduler.class, args); } @@ -34,6 +35,7 @@ public class InitDolphinScheduler { @Component @Profile("init") static class InitRunner implements CommandLineRunner { + private static final Logger logger = LoggerFactory.getLogger(InitRunner.class); private final DolphinSchedulerManager dolphinSchedulerManager; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java index 22e1338431..bdd7fcf997 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Component; @SpringBootApplication public class UpgradeDolphinScheduler { + public static void main(String[] args) { SpringApplication.run(UpgradeDolphinScheduler.class, args); } @@ -34,6 +35,7 @@ public class UpgradeDolphinScheduler { @Component @Profile("upgrade") static class UpgradeRunner implements CommandLineRunner { + private static final Logger logger = LoggerFactory.getLogger(UpgradeRunner.class); private final DolphinSchedulerManager dolphinSchedulerManager; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java index 14fa45cff1..7361932034 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java @@ -32,6 +32,7 @@ import org.springframework.stereotype.Service; @Service public class MySQLUpgradeDao extends UpgradeDao { + public static final Logger logger = LoggerFactory.getLogger(MySQLUpgradeDao.class); private MySQLUpgradeDao(DataSource dataSource) { @@ -62,8 +63,8 @@ public class MySQLUpgradeDao extends UpgradeDao { rs = conn.getMetaData().getTables(conn.getCatalog(), conn.getSchema(), tableName, null); return rs.next(); } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(rs, conn); } @@ -77,16 +78,16 @@ public class MySQLUpgradeDao extends UpgradeDao { * @return if column name exist return true，else return false */ @Override - public boolean isExistsColumn(String tableName,String columnName) { + public boolean isExistsColumn(String tableName, String columnName) { Connection conn = null; try { conn = dataSource.getConnection(); - ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(),tableName,columnName); + ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(), tableName, columnName); return rs.next(); } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(conn); } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java index f0ad67c1fd..43761ba564 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java @@ -33,6 +33,7 @@ import org.springframework.stereotype.Service; @Service public class PostgreSQLUpgradeDao extends UpgradeDao { + public static final Logger logger = LoggerFactory.getLogger(PostgreSQLUpgradeDao.class); private PostgreSQLUpgradeDao(DataSource dataSource) { @@ -71,7 +72,6 @@ public class PostgreSQLUpgradeDao extends UpgradeDao { return ""; } - /** * determines whether a table exists * diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java index d397794166..48a46249e7 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java @@ -17,17 +17,17 @@ package org.apache.dolphinscheduler.tools.datasource.dao; -import java.sql.SQLException; -import java.util.Objects; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +39,7 @@ import com.google.common.base.Strings; * resource dao */ public class ResourceDao { + public static final Logger logger = LoggerFactory.getLogger(ResourceDao.class); /** @@ -82,7 +83,8 @@ public class ResourceDao { private Map listAllResourcesByFileType(Connection conn, int type) { Map resourceSizeMap = new HashMap<>(); - String sql = String.format("SELECT full_name, type, size, is_directory FROM t_ds_resources where type = %d", type); + String sql = + String.format("SELECT full_name, type, size, is_directory FROM t_ds_resources where type = %d", type); ResultSet rs = null; PreparedStatement pstmt = null; try { @@ -97,7 +99,7 @@ public class ResourceDao { if (StringUtils.isNotBlank(fullName) && !isDirectory) { String[] splits = fullName.split("/"); for (int i = 1; i < splits.length; i++) { - String parentFullName = Joiner.on("/").join(Arrays.copyOfRange(splits,0, splits.length - i)); + String parentFullName = Joiner.on("/").join(Arrays.copyOfRange(splits, 0, splits.length - i)); if (!Strings.isNullOrEmpty(parentFullName)) { long size = resourceSizeMap.getOrDefault(parentFullName, 0L); resourceSizeMap.put(parentFullName, size + fileSize); diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index ed8dc6fcd9..fb6d32fa49 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -15,73 +15,73 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + dev-SNAPSHOT + - - - dolphinscheduler - org.apache.dolphinscheduler - dev-SNAPSHOT - - 4.0.0 + dolphinscheduler-ui - dolphinscheduler-ui + ${project.artifactId} - ${project.artifactId} + + v16.13.1 + v6.32.6 + src + 1.12.1 + - - v16.13.1 - v6.32.6 - src - 1.12.1 - + + + release + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + false + + + + install node and pnpm + + install-node-and-pnpm + + + ${node.version} + ${pnpm.version} + + + + pnpm install + + pnpm + + generate-resources + + install + + + + pnpm run build:prod + + pnpm + + + run build:prod + + + + + - - - release - - - - com.github.eirslett - frontend-maven-plugin - ${frontend-maven-plugin.version} - - false - - - - install node and pnpm - - install-node-and-pnpm - - - ${node.version} - ${pnpm.version} - - - - pnpm install - - pnpm - - generate-resources - - install - - - - pnpm run build:prod - - pnpm - - - run build:prod - - - - - - - - - + + + diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 8f8a27372b..c3ae0884b6 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker; -import org.apache.commons.collections4.CollectionUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -35,6 +34,13 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collection; + +import javax.annotation.PostConstruct; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -44,9 +50,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; -import javax.annotation.PostConstruct; -import java.util.Collection; - @SpringBootApplication @EnableTransactionManagement @ComponentScan(basePackages = "org.apache.dolphinscheduler", excludeFilters = { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java index 58e1fe7e88..eb4cbf1210 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java @@ -17,25 +17,28 @@ package org.apache.dolphinscheduler.server.worker.message; -import lombok.NonNull; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.remote.command.BaseCommand; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.service.utils.LoggerUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.PostConstruct; + +import lombok.NonNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Component; + @Component public class MessageRetryRunner extends BaseDaemonThread { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java index a7d05a4791..3cdabad3de 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java @@ -44,10 +44,10 @@ public class TaskExecuteResultMessageSender implements MessageSender public TaskRejectCommand buildMessage(TaskExecutionContext taskExecutionContext, String masterAddress) { TaskRejectCommand taskRejectMessage = new TaskRejectCommand(workerConfig.getWorkerAddress(), - masterAddress, - System.currentTimeMillis()); + masterAddress, + System.currentTimeMillis()); taskRejectMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskRejectMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskRejectMessage.setHost(taskExecutionContext.getHost()); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java index 6d3de8e146..32213774b4 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.server.worker.metrics; - import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; @UtilityClass public class TaskMetrics { @@ -45,8 +44,7 @@ public class TaskMetrics { Counter.builder("ds.task.execution.count.by.type") .tag("task_type", taskChannelFactory.getName()) .description("task execution counter by type") - .register(Metrics.globalRegistry) - ); + .register(Metrics.globalRegistry)); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java index abe5f3fbb2..ad40bc3443 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java @@ -20,25 +20,25 @@ package org.apache.dolphinscheduler.server.worker.metrics; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; -import lombok.experimental.UtilityClass; @UtilityClass public class WorkerServerMetrics { private final Counter workerOverloadCounter = - Counter.builder("ds.worker.overload.count") - .description("overloaded workers count") - .register(Metrics.globalRegistry); + Counter.builder("ds.worker.overload.count") + .description("overloaded workers count") + .register(Metrics.globalRegistry); private final Counter workerFullSubmitQueueCounter = - Counter.builder("ds.worker.full.submit.queue.count") - .description("full worker submit queues count") - .register(Metrics.globalRegistry); + Counter.builder("ds.worker.full.submit.queue.count") + .description("full worker submit queues count") + .register(Metrics.globalRegistry); private final Counter workerResourceDownloadSuccessCounter = Counter.builder("ds.worker.resource.download.count") @@ -61,11 +61,11 @@ public class WorkerServerMetrics { private final DistributionSummary workerResourceDownloadSizeDistribution = DistributionSummary.builder("ds.worker.resource.download.size") - .baseUnit("bytes") - .publishPercentiles(0.5, 0.75, 0.95, 0.99) - .publishPercentileHistogram() - .description("size of downloaded resource files on worker") - .register(Metrics.globalRegistry); + .baseUnit("bytes") + .publishPercentiles(0.5, 0.75, 0.95, 0.99) + .publishPercentileHistogram() + .description("size of downloaded resource files on worker") + .register(Metrics.globalRegistry); public void incWorkerOverloadCount() { workerOverloadCounter.increment(); @@ -93,8 +93,8 @@ public class WorkerServerMetrics { public void registerWorkerRunningTaskGauge(final Supplier supplier) { Gauge.builder("ds.task.running", supplier) - .description("number of running tasks on workers") - .register(Metrics.globalRegistry); + .description("number of running tasks on workers") + .register(Metrics.globalRegistry); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java index d36c80012d..0f6a3d093b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java @@ -30,7 +30,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -48,7 +47,7 @@ public class HostUpdateProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_REQUEST == command.getType(), - String.format("invalid command type : %s", command.getType())); + String.format("invalid command type : %s", command.getType())); HostUpdateCommand updateCommand = JSONUtils.parseObject(command.getBody(), HostUpdateCommand.class); if (updateCommand == null) { logger.error("host update command is null"); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java index d85181f1ed..ec5bd2ebad 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java @@ -37,6 +37,7 @@ import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.utils.LogUtils; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -115,8 +116,9 @@ public class TaskDispatchProcessor implements NettyRequestProcessor { taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); // delay task process - long remainTime = DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - taskExecutionContext.getDelayTime() * 60L); + long remainTime = + DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), + taskExecutionContext.getDelayTime() * 60L); if (remainTime > 0) { logger.info("Current taskInstance is choose delay execution, delay time: {}s", remainTime); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.DELAY_EXECUTION); @@ -137,10 +139,14 @@ public class TaskDispatchProcessor implements NettyRequestProcessor { // submit task to manager boolean offer = workerManager.offer(workerTaskExecuteRunnable); if (!offer) { - logger.warn("submit task to wait queue error, queue is full, current queue size is {}, will send a task reject message to master", workerManager.getWaitSubmitQueueSize()); - workerMessageSender.sendMessageWithRetry(taskExecutionContext, workflowMasterAddress, CommandType.TASK_REJECT); + logger.warn( + "submit task to wait queue error, queue is full, current queue size is {}, will send a task reject message to master", + workerManager.getWaitSubmitQueueSize()); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, workflowMasterAddress, + CommandType.TASK_REJECT); } else { - logger.info("Submit task to wait queue success, current queue size is {}", workerManager.getWaitSubmitQueueSize()); + logger.info("Submit task to wait queue success, current queue size is {}", + workerManager.getWaitSubmitQueueSize()); } } finally { LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java index 4f6eb9ce47..e2ff1d005a 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -26,11 +24,15 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + /** * task execute running ack, from master to worker */ diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java index fc8ead96fa..9cbde24d9e 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java @@ -24,13 +24,13 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckMessage; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java index 7fefb61729..68c88332e2 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -26,11 +24,15 @@ import org.apache.dolphinscheduler.remote.command.TaskRejectAckCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + @Component public class TaskRejectAckProcessor implements NettyRequestProcessor { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java index b264eb0110..3190af4b4f 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java @@ -17,10 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -34,11 +30,18 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; + /** * task save point processor */ @@ -62,8 +65,9 @@ public class TaskSavePointProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_SAVEPOINT_REQUEST == command.getType(), - String.format("invalid command type : %s", command.getType())); - TaskSavePointRequestCommand taskSavePointRequestCommand = JSONUtils.parseObject(command.getBody(), TaskSavePointRequestCommand.class); + String.format("invalid command type : %s", command.getType())); + TaskSavePointRequestCommand taskSavePointRequestCommand = + JSONUtils.parseObject(command.getBody(), TaskSavePointRequestCommand.class); if (taskSavePointRequestCommand == null) { logger.error("task savepoint request command is null"); return; @@ -71,9 +75,11 @@ public class TaskSavePointProcessor implements NettyRequestProcessor { logger.info("Receive task savepoint command : {}", taskSavePointRequestCommand); int taskInstanceId = taskSavePointRequestCommand.getTaskInstanceId(); - TaskExecutionContext taskExecutionContext = TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); + TaskExecutionContext taskExecutionContext = + TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); if (taskExecutionContext == null) { - logger.error("taskRequest cache is null, taskInstanceId: {}", taskSavePointRequestCommand.getTaskInstanceId()); + logger.error("taskRequest cache is null, taskInstanceId: {}", + taskSavePointRequestCommand.getTaskInstanceId()); return; } @@ -91,12 +97,15 @@ public class TaskSavePointProcessor implements NettyRequestProcessor { TaskSavePointResponseCommand taskSavePointResponseCommand = new TaskSavePointResponseCommand(); taskSavePointResponseCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); channel.writeAndFlush(taskSavePointResponseCommand.convert2Command()).addListener(new ChannelFutureListener() { + @Override public void operationComplete(ChannelFuture future) throws Exception { if (!future.isSuccess()) { - logger.error("Submit kill response to master error, kill command: {}", taskSavePointResponseCommand); + logger.error("Submit kill response to master error, kill command: {}", + taskSavePointResponseCommand); } else - logger.info("Submit kill response to master success, kill command: {}", taskSavePointResponseCommand); + logger.info("Submit kill response to master success, kill command: {}", + taskSavePointResponseCommand); } }); } @@ -117,7 +126,7 @@ public class TaskSavePointProcessor implements NettyRequestProcessor { return; } try { - ((StreamTask)task).savePoint(); + ((StreamTask) task).savePoint(); } catch (Exception e) { logger.error("task save point error", e); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java index 36b95e1b62..a8f8cad6bc 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.server.worker.registry; -import lombok.NonNull; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java index 08d434ea2d..7e7e294ecc 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.worker.registry; import org.apache.dolphinscheduler.registry.api.StrategyType; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java index 203e5711f1..2d1c3359b2 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java @@ -27,14 +27,15 @@ import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcServer; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import java.time.Duration; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import java.time.Duration; - @Service @ConditionalOnProperty(prefix = "worker.registry-disconnect-strategy", name = "strategy", havingValue = "waiting") public class WorkerWaitingStrategy implements WorkerConnectStrategy { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java index 5a2752ca34..ae2fba6c7b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java @@ -29,14 +29,14 @@ import java.util.Map; import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; -import lombok.NonNull; - @Component public class WorkerMessageSender { @@ -54,7 +54,7 @@ public class WorkerMessageSender { public void init() { Map messageSenders = applicationContext.getBeansOfType(MessageSender.class); messageSenders.values().forEach(messageSender -> messageSenderMap.put(messageSender.getMessageType(), - messageSender)); + messageSender)); } // todo: use message rather than context diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java index 4be20873f9..a01cbaabdf 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java @@ -57,7 +57,7 @@ public class WorkerRpcClient implements AutoCloseable { this.nettyRemotingClient = new NettyRemotingClient(nettyClientConfig); // we only use the client to handle the ack message, we can optimize this, send ack to the nettyServer. this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK, - taskExecuteRunningAckProcessor); + taskExecuteRunningAckProcessor); this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RESULT_ACK, taskExecuteResultAckProcessor); this.nettyRemotingClient.registerProcessor(CommandType.TASK_REJECT_ACK, taskRejectAckProcessor); logger.info("Worker rpc client started"); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java index c00fb4f99a..e926b8d9bb 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -29,6 +28,8 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; +import lombok.NonNull; + public class DefaultWorkerDelayTaskExecuteRunnable extends WorkerDelayTaskExecuteRunnable { public DefaultWorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, @@ -38,7 +39,8 @@ public class DefaultWorkerDelayTaskExecuteRunnable extends WorkerDelayTaskExecut @NonNull AlertClientService alertClientService, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate) { - super(taskExecutionContext, workerConfig, workflowMaster, workerMessageSender, alertClientService, taskPluginManager, storageOperate); + super(taskExecutionContext, workerConfig, workflowMaster, workerMessageSender, alertClientService, + taskPluginManager, storageOperate); } @Override diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java index 49262b3900..dad1421675 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -27,7 +26,11 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; -public class DefaultWorkerDelayTaskExecuteRunnableFactory extends WorkerDelayTaskExecuteRunnableFactory { +import lombok.NonNull; + +public class DefaultWorkerDelayTaskExecuteRunnableFactory + extends + WorkerDelayTaskExecuteRunnableFactory { protected DefaultWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @@ -36,7 +39,8 @@ public class DefaultWorkerDelayTaskExecuteRunnableFactory extends WorkerDelayTas @NonNull AlertClientService alertClientService, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate) { - super(taskExecutionContext, workerConfig, workflowMasterAddress, workerMessageSender, alertClientService, taskPluginManager, storageOperate); + super(taskExecutionContext, workerConfig, workflowMasterAddress, workerMessageSender, alertClientService, + taskPluginManager, storageOperate); } @Override diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java index 19351ee764..e322aa9cf0 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -26,10 +25,13 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; -import javax.annotation.Nullable; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +import lombok.NonNull; + public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRunnable implements Delayed { protected WorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, @@ -39,7 +41,8 @@ public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRu @NonNull AlertClientService alertClientService, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate) { - super(taskExecutionContext, workerConfig, masterAddress, workerMessageSender, alertClientService, taskPluginManager, storageOperate); + super(taskExecutionContext, workerConfig, masterAddress, workerMessageSender, alertClientService, + taskPluginManager, storageOperate); } @Override @@ -48,7 +51,8 @@ public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRu return unit.convert( DateUtils.getRemainTime( DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - taskExecutionContext.getDelayTime() * 60L), TimeUnit.SECONDS); + taskExecutionContext.getDelayTime() * 60L), + TimeUnit.SECONDS); } @Override diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java index 7db96fc380..d8677b4680 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -27,7 +26,11 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; -public abstract class WorkerDelayTaskExecuteRunnableFactory implements WorkerTaskExecuteRunnableFactory { +import lombok.NonNull; + +public abstract class WorkerDelayTaskExecuteRunnableFactory + implements + WorkerTaskExecuteRunnableFactory { protected final @NonNull TaskExecutionContext taskExecutionContext; protected final @NonNull WorkerConfig workerConfig; @@ -38,13 +41,13 @@ public abstract class WorkerDelayTaskExecuteRunnableFactory taskExecuteThreadMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap taskExecuteThreadMap = + new ConcurrentHashMap<>(); public WorkerManagerThread(WorkerConfig workerConfig) { workerExecThreads = workerConfig.getExecThreads(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java index ebc358b026..d0df1dcb31 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java @@ -45,11 +45,9 @@ import org.apache.dolphinscheduler.service.utils.CommonUtils; import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.nio.file.NoSuchFileException; -import java.util.Date; import java.util.List; import javax.annotation.Nullable; @@ -92,11 +90,12 @@ public abstract class WorkerTaskExecuteRunnable implements Runnable { this.alertClientService = alertClientService; this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; - String taskLogName = LoggerUtils.buildTaskId(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - taskExecutionContext.getProcessDefineCode(), - taskExecutionContext.getProcessDefineVersion(), - taskExecutionContext.getProcessInstanceId(), - taskExecutionContext.getTaskInstanceId()); + String taskLogName = + LoggerUtils.buildTaskId(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), + taskExecutionContext.getProcessDefineCode(), + taskExecutionContext.getProcessDefineVersion(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); taskExecutionContext.setTaskLogName(taskLogName); logger.info("Set task logger name: {}", taskLogName); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java index fee1635be0..8ec586397f 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; -import lombok.experimental.UtilityClass; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -28,6 +26,9 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager; import javax.annotation.Nullable; +import lombok.NonNull; +import lombok.experimental.UtilityClass; + @UtilityClass public class WorkerTaskExecuteRunnableFactoryBuilder { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java index d0467a25dc..1aea96d9d8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java @@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; import org.apache.dolphinscheduler.service.storage.StorageOperate; -import org.apache.dolphinscheduler.service.utils.CommonUtils; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.SystemUtils; diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java index 93dcc9fee4..2323e591bf 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.worker.config; import org.apache.dolphinscheduler.service.alert.AlertClientService; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,4 +37,4 @@ public class BeanConfigTest { public void alertClientService() { Assertions.assertNotNull(alertClientService); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java index 049d01ac29..c2e0ce4d8a 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -28,6 +27,7 @@ import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -35,6 +35,8 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import io.netty.channel.Channel; + /** * test task execute processor */ diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java index f0189df876..929c80a62c 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java @@ -25,8 +25,9 @@ import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; -import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; public class DefaultWorkerDelayTaskExecuteRunnableTest { @@ -61,8 +62,7 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { workerMessageSender, alertClientService, taskPluginManager, - storageOperate - ); + storageOperate); Assertions.assertAll(workerTaskExecuteRunnable::run); Assertions.assertEquals(TaskExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); @@ -78,7 +78,8 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { .firstSubmitTime(System.currentTimeMillis()) .taskLogName("TestLogName") .taskType("SQL") - .taskParams("{\"localParams\":[],\"resourceList\":[],\"type\":\"POSTGRESQL\",\"datasource\":null,\"sql\":\"select * from t_ds_user\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10,\"conditionResult\":\"null\",\"dependence\":\"null\",\"switchResult\":\"null\",\"waitStartTimeout\":null}") + .taskParams( + "{\"localParams\":[],\"resourceList\":[],\"type\":\"POSTGRESQL\",\"datasource\":null,\"sql\":\"select * from t_ds_user\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10,\"conditionResult\":\"null\",\"dependence\":\"null\",\"switchResult\":\"null\",\"waitStartTimeout\":null}") .build(); WorkerTaskExecuteRunnable workerTaskExecuteRunnable = new DefaultWorkerDelayTaskExecuteRunnable( taskExecutionContext, @@ -87,8 +88,7 @@ public class DefaultWorkerDelayTaskExecuteRunnableTest { workerMessageSender, alertClientService, taskPluginManager, - storageOperate - ); + storageOperate); Assertions.assertAll(workerTaskExecuteRunnable::run); Assertions.assertEquals(TaskExecutionStatus.FAILURE, taskExecutionContext.getCurrentExecutionStatus()); diff --git a/pom.xml b/pom.xml index e368b4f5a2..87f5b5e723 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ 3.1.1 3.2.1 2.2.0 - 2.23.0 + 2.27.2 0.8.8 false 2.7 @@ -639,9 +639,12 @@ spotless-maven-plugin ${spotless.version} - - HEAD + + + **/e2e/**/*.java + **/microbench/**/*.java + style/spotless_dolphinscheduler_formatter.xml @@ -687,8 +690,12 @@ - **/*.md + docs/**/*.md + + + **/.github/**/*.md + diff --git a/tools/release/README.md b/tools/release/README.md index b4596f0f88..d2bd5ca61a 100644 --- a/tools/release/README.md +++ b/tools/release/README.md @@ -36,3 +36,4 @@ python release.py -h * Auto cherry-pick: `python release.py cherry-pick`, will cause error when your default branch is not up-to-date, or cherry-pick with conflict. But if you fix you can directly re-run this command, it will continue the pick * Generate changelog: `python release.py changelog` * Generate contributor: `python release.py contributor` +