Browse Source

Clean up third-party files that are not mentioned in LICENSE (#6165)

2.0.7-release
kezhenxu94 3 years ago committed by GitHub
parent
commit
ea2a9dbcb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 37
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java
  2. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java
  3. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java
  4. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java
  5. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java
  6. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
  7. 89
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
  8. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java
  9. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java
  10. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java
  11. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
  12. 7
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java
  13. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
  14. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
  15. 14
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
  16. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  17. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
  18. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  19. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
  20. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java
  21. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
  22. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java
  23. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
  24. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
  25. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
  26. 9
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
  27. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  28. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
  29. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
  30. 23
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java
  31. 34
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java
  32. 226
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java
  33. 5
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
  34. 3
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  35. 5
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java
  36. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java
  37. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  38. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java
  39. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java
  40. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java
  41. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java
  42. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java
  43. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java
  44. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java
  45. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java
  46. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java
  47. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
  48. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java
  49. 9
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java
  50. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java
  51. 118
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java
  52. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
  53. 25
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java
  54. 33
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java
  55. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
  56. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  57. 99
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  58. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
  59. 185
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
  60. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  61. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  62. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  63. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  64. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java
  65. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
  66. 310
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
  67. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
  68. 226
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  69. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
  70. 26
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java
  71. 39
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java
  72. 96
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java
  73. 3
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java
  74. 9
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java
  75. 7
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
  76. 5
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java
  77. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
  78. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
  79. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java
  80. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
  81. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
  82. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
  83. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
  84. 69
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
  85. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
  86. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
  87. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
  88. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java
  89. 300
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
  90. 7
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
  91. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
  92. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
  93. 14
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
  94. 5
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
  95. 48
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
  96. 5
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  97. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
  98. 3
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java
  99. 3
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
  100. 56
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
  101. Some files were not shown because too many files have changed in this diff Show More

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java

@ -16,25 +16,44 @@
*/ */
package org.apache.dolphinscheduler.api.configuration; package org.apache.dolphinscheduler.api.configuration;
import com.google.common.collect.Multimap; import static com.google.common.collect.Maps.newTreeMap;
import io.swagger.models.*;
import io.swagger.models.parameters.Parameter; import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
import org.springframework.context.MessageSource; import org.springframework.context.MessageSource;
import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Primary;
import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import com.google.common.collect.Multimap;
import io.swagger.models.Contact;
import io.swagger.models.Info;
import io.swagger.models.Model;
import io.swagger.models.Scheme;
import io.swagger.models.Swagger;
import io.swagger.models.Tag;
import io.swagger.models.parameters.Parameter;
import springfox.documentation.service.ApiInfo; import springfox.documentation.service.ApiInfo;
import springfox.documentation.service.ApiListing; import springfox.documentation.service.ApiListing;
import springfox.documentation.service.Documentation; import springfox.documentation.service.Documentation;
import springfox.documentation.service.ResourceListing; import springfox.documentation.service.ResourceListing;
import springfox.documentation.swagger2.mappers.*; import springfox.documentation.swagger2.mappers.LicenseMapper;
import springfox.documentation.swagger2.mappers.ModelMapper;
import java.util.*; import springfox.documentation.swagger2.mappers.ParameterMapper;
import springfox.documentation.swagger2.mappers.SecurityMapper;
import static com.google.common.collect.Maps.newTreeMap; import springfox.documentation.swagger2.mappers.ServiceModelToSwagger2Mapper;
import springfox.documentation.swagger2.mappers.VendorExtensionsMapper;
/** /**
* application configuration * application configuration

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java

@ -25,7 +25,8 @@ import static org.apache.dolphinscheduler.common.Constants.HTTP_X_REAL_IP;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.HashMap; import java.util.HashMap;

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java

@ -28,10 +28,10 @@ import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.api.service.SessionService; import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang.StringUtils;
import java.util.Map; import java.util.Map;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java

@ -43,10 +43,11 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -588,7 +589,7 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam("codes") String codes) { @RequestParam("codes") String codes) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
List<String> deleteFailedCodeList = new ArrayList<>(); List<String> deleteFailedCodeList = new ArrayList<>();
if (StringUtils.isNotEmpty(codes)) { if (!StringUtils.isEmpty(codes)) {
String[] processDefinitionCodeArray = codes.split(","); String[] processDefinitionCodeArray = codes.split(",");
for (String strProcessDefinitionCode : processDefinitionCodeArray) { for (String strProcessDefinitionCode : processDefinitionCodeArray) {
long code = Long.parseLong(strProcessDefinitionCode); long code = Long.parseLong(strProcessDefinitionCode);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java

@ -37,10 +37,11 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -395,7 +396,7 @@ public class ProcessInstanceController extends BaseController {
// task queue // task queue
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
List<String> deleteFailedIdList = new ArrayList<>(); List<String> deleteFailedIdList = new ArrayList<>();
if (StringUtils.isNotEmpty(processInstanceIds)) { if (!StringUtils.isEmpty(processInstanceIds)) {
String[] processInstanceIdArray = processInstanceIds.split(","); String[] processInstanceIdArray = processInstanceIds.split(",");
for (String strProcessInstanceId : processInstanceIdArray) { for (String strProcessInstanceId : processInstanceIdArray) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@ -52,9 +52,10 @@ import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.util.Map; import java.util.Map;
import org.slf4j.Logger; import org.slf4j.Logger;

89
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java

@ -21,11 +21,11 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.security.Authenticator; import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang.StringUtils;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
@ -36,56 +36,57 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.servlet.HandlerInterceptor; import org.springframework.web.servlet.HandlerInterceptor;
/** /**
* login interceptor, must login first * login interceptor, must log in first
*/ */
public class LoginHandlerInterceptor implements HandlerInterceptor { public class LoginHandlerInterceptor implements HandlerInterceptor {
private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptor.class); private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptor.class);
@Autowired @Autowired
private UserMapper userMapper; private UserMapper userMapper;
@Autowired @Autowired
private Authenticator authenticator; private Authenticator authenticator;
/** /**
* Intercept the execution of a handler. Called after HandlerMapping determined * Intercept the execution of a handler. Called after HandlerMapping determined
* @param request current HTTP request *
* @param response current HTTP response * @param request current HTTP request
* @param handler chosen handler to execute, for type and/or instance evaluation * @param response current HTTP response
* @return boolean true or false * @param handler chosen handler to execute, for type and/or instance evaluation
*/ * @return boolean true or false
@Override */
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { @Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
// get token // get token
String token = request.getHeader("token"); String token = request.getHeader("token");
User user = null; User user;
if (StringUtils.isEmpty(token)){ if (StringUtils.isEmpty(token)) {
user = authenticator.getAuthUser(request); user = authenticator.getAuthUser(request);
// if user is null // if user is null
if (user == null) { if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED); response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("user does not exist"); logger.info("user does not exist");
return false; return false;
} }
}else { } else {
user = userMapper.queryUserByToken(token); user = userMapper.queryUserByToken(token);
if (user == null) { if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED); response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("user token has expired"); logger.info("user token has expired");
return false; return false;
} }
} }
// check user state // check user state
if (user.getState() == Flag.NO.ordinal()) { if (user.getState() == Flag.NO.ordinal()) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED); response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info(Status.USER_DISABLED.getMsg()); logger.info(Status.USER_DISABLED.getMsg());
return false; return false;
} }
request.setAttribute(Constants.SESSION_USER, user); request.setAttribute(Constants.SESSION_USER, user);
return true; return true;
} }
} }

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java

@ -18,9 +18,9 @@
package org.apache.dolphinscheduler.api.interceptor; package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration; import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@ -74,7 +74,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
// tenant-level rate limit // tenant-level rate limit
if (trafficConfiguration.isTrafficTenantControlSwitch()) { if (trafficConfiguration.isTrafficTenantControlSwitch()) {
String token = request.getHeader("token"); String token = request.getHeader("token");
if (StringUtils.isNotEmpty(token)) { if (!StringUtils.isEmpty(token)) {
RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token); RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token);
if (!tenantRateLimiter.tryAcquire()) { if (!tenantRateLimiter.tryAcquire()) {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java

@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.api.security;
import org.apache.dolphinscheduler.api.security.impl.ldap.LdapAuthenticator; import org.apache.dolphinscheduler.api.security.impl.ldap.LdapAuthenticator;
import org.apache.dolphinscheduler.api.security.impl.pwd.PasswordAuthenticator; import org.apache.dolphinscheduler.api.security.impl.pwd.PasswordAuthenticator;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java

@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.api.service.AlertGroupService;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.vo.AlertGroupVo; import org.apache.dolphinscheduler.dao.vo.AlertGroupVo;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -195,7 +195,7 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
Date now = new Date(); Date now = new Date();
if (StringUtils.isNotEmpty(groupName)) { if (!StringUtils.isEmpty(groupName)) {
alertGroup.setGroupName(groupName); alertGroup.setGroupName(groupName);
} }
alertGroup.setDescription(desc); alertGroup.setDescription(desc);
@ -248,6 +248,6 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
*/ */
@Override @Override
public boolean existGroupName(String groupName) { public boolean existGroupName(String groupName) {
return BooleanUtils.isTrue(alertGroupMapper.existGroupName(groupName)); return alertGroupMapper.existGroupName(groupName) == Boolean.TRUE;
} }
} }

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java

@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO; import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
@ -85,7 +84,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
if (BooleanUtils.isTrue(alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()))) { if (alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()) == Boolean.TRUE) {
putMsg(result, Status.PLUGIN_INSTANCE_ALREADY_EXIT); putMsg(result, Status.PLUGIN_INSTANCE_ALREADY_EXIT);
return result; return result;
} }
@ -184,7 +183,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A
@Override @Override
public boolean checkExistPluginInstanceName(String pluginInstanceName) { public boolean checkExistPluginInstanceName(String pluginInstanceName) {
return BooleanUtils.isTrue(alertPluginInstanceMapper.existInstanceName(pluginInstanceName)); return alertPluginInstanceMapper.existInstanceName(pluginInstanceName) == Boolean.TRUE;
} }
@Override @Override

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java

@ -24,9 +24,10 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.Date; import java.util.Date;
@ -157,7 +158,7 @@ public class BaseServiceImpl implements BaseService {
public Map<String, Object> checkAndParseDateParameters(String startDateStr, String endDateStr) { public Map<String, Object> checkAndParseDateParameters(String startDateStr, String endDateStr) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
Date start = null; Date start = null;
if (StringUtils.isNotEmpty(startDateStr)) { if (!StringUtils.isEmpty(startDateStr)) {
start = DateUtils.getScheduleDate(startDateStr); start = DateUtils.getScheduleDate(startDateStr);
if (Objects.isNull(start)) { if (Objects.isNull(start)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.START_END_DATE); putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.START_END_DATE);
@ -167,7 +168,7 @@ public class BaseServiceImpl implements BaseService {
result.put(Constants.START_TIME, start); result.put(Constants.START_TIME, start);
Date end = null; Date end = null;
if (StringUtils.isNotEmpty(endDateStr)) { if (!StringUtils.isEmpty(endDateStr)) {
end = DateUtils.getScheduleDate(endDateStr); end = DateUtils.getScheduleDate(endDateStr);
if (Objects.isNull(end)) { if (Objects.isNull(end)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.START_END_DATE); putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.START_END_DATE);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java

@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TriFunction; import org.apache.dolphinscheduler.common.utils.TriFunction;
import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser;
@ -43,6 +42,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
@ -144,7 +145,7 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
Date start = null; Date start = null;
Date end = null; Date end = null;
if (StringUtils.isNotEmpty(startDate) && StringUtils.isNotEmpty(endDate)) { if (!StringUtils.isEmpty(startDate) && !StringUtils.isEmpty(endDate)) {
start = DateUtils.getScheduleDate(startDate); start = DateUtils.getScheduleDate(startDate);
end = DateUtils.getScheduleDate(endDate); end = DateUtils.getScheduleDate(endDate);
if (Objects.isNull(start) || Objects.isNull(end)) { if (Objects.isNull(start) || Objects.isNull(end)) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@ -28,12 +28,13 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation; import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@ -37,6 +36,7 @@ import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapp
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.commons.collections4.SetUtils; import org.apache.commons.collections4.SetUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -126,11 +126,11 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
} }
if (environmentMapper.insert(env) > 0) { if (environmentMapper.insert(env) > 0) {
if (StringUtils.isNotEmpty(workerGroups)) { if (!StringUtils.isEmpty(workerGroups)) {
List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){}); List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){});
if (CollectionUtils.isNotEmpty(workerGroupList)) { if (CollectionUtils.isNotEmpty(workerGroupList)) {
workerGroupList.stream().forEach(workerGroup -> { workerGroupList.stream().forEach(workerGroup -> {
if (StringUtils.isNotEmpty(workerGroup)) { if (!StringUtils.isEmpty(workerGroup)) {
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(env.getCode()); relation.setEnvironmentCode(env.getCode());
relation.setWorkerGroup(workerGroup); relation.setWorkerGroup(workerGroup);
@ -338,7 +338,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
} }
Set<String> workerGroupSet; Set<String> workerGroupSet;
if (StringUtils.isNotEmpty(workerGroups)) { if (!StringUtils.isEmpty(workerGroups)) {
workerGroupSet = JSONUtils.parseObject(workerGroups, new TypeReference<Set<String>>() {}); workerGroupSet = JSONUtils.parseObject(workerGroups, new TypeReference<Set<String>>() {});
} else { } else {
workerGroupSet = new TreeSet<>(); workerGroupSet = new TreeSet<>();
@ -370,14 +370,14 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
int update = environmentMapper.update(env, new UpdateWrapper<Environment>().lambda().eq(Environment::getCode,code)); int update = environmentMapper.update(env, new UpdateWrapper<Environment>().lambda().eq(Environment::getCode,code));
if (update > 0) { if (update > 0) {
deleteWorkerGroupSet.stream().forEach(key -> { deleteWorkerGroupSet.stream().forEach(key -> {
if (StringUtils.isNotEmpty(key)) { if (!StringUtils.isEmpty(key)) {
relationMapper.delete(new QueryWrapper<EnvironmentWorkerGroupRelation>() relationMapper.delete(new QueryWrapper<EnvironmentWorkerGroupRelation>()
.lambda() .lambda()
.eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code)); .eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code));
} }
}); });
addWorkerGroupSet.stream().forEach(key -> { addWorkerGroupSet.stream().forEach(key -> {
if (StringUtils.isNotEmpty(key)) { if (!StringUtils.isEmpty(key)) {
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(code); relation.setEnvironmentCode(code);
relation.setWorkerGroup(key); relation.setWorkerGroup(key);
@ -448,7 +448,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
putMsg(result, Status.ENVIRONMENT_CONFIG_IS_NULL); putMsg(result, Status.ENVIRONMENT_CONFIG_IS_NULL);
return result; return result;
} }
if (StringUtils.isNotEmpty(workerGroups)) { if (!StringUtils.isEmpty(workerGroups)) {
List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){}); List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){});
if (Objects.isNull(workerGroupList)) { if (Objects.isNull(workerGroupList)) {
putMsg(result, Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID); putMsg(result, Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID);

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@ -42,7 +42,6 @@ import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@ -59,6 +58,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -416,7 +416,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
//To add startParams only when repeat running is needed //To add startParams only when repeat running is needed
Map<String, Object> cmdParam = new HashMap<>(); Map<String, Object> cmdParam = new HashMap<>();
cmdParam.put(CMD_PARAM_RECOVER_PROCESS_ID_STRING, instanceId); cmdParam.put(CMD_PARAM_RECOVER_PROCESS_ID_STRING, instanceId);
if (StringUtils.isNotEmpty(startParams)) { if (!StringUtils.isEmpty(startParams)) {
cmdParam.put(CMD_PARAM_START_PARAMS, startParams); cmdParam.put(CMD_PARAM_START_PARAMS, startParams);
} }
@ -527,7 +527,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
command.setFailureStrategy(failureStrategy); command.setFailureStrategy(failureStrategy);
} }
if (StringUtils.isNotEmpty(startNodeList)) { if (!StringUtils.isEmpty(startNodeList)) {
cmdParam.put(CMD_PARAM_START_NODE_NAMES, startNodeList); cmdParam.put(CMD_PARAM_START_NODE_NAMES, startNodeList);
} }
if (warningType != null) { if (warningType != null) {
@ -545,7 +545,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
Date start = null; Date start = null;
Date end = null; Date end = null;
if (StringUtils.isNotEmpty(schedule)) { if (!StringUtils.isEmpty(schedule)) {
String[] interval = schedule.split(","); String[] interval = schedule.split(",");
if (interval.length == 2) { if (interval.length == 2) {
start = DateUtils.getScheduleDate(interval[0]); start = DateUtils.getScheduleDate(interval[0]);

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java

@ -23,12 +23,13 @@ import org.apache.dolphinscheduler.api.service.LoggerService;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.ArrayUtils; import org.apache.dolphinscheduler.common.utils.ArrayUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Objects; import java.util.Objects;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -44,7 +44,6 @@ import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
@ -71,6 +70,8 @@ import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.permission.PermissionCheck; import org.apache.dolphinscheduler.service.permission.PermissionCheck;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@ -49,7 +49,6 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@ -68,6 +67,8 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
@ -653,7 +654,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()); taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion());
String localParams = JSONUtils.getNodeString(taskDefinitionLog.getTaskParams(), LOCAL_PARAMS); String localParams = JSONUtils.getNodeString(taskDefinitionLog.getTaskParams(), LOCAL_PARAMS);
if (StringUtils.isNotEmpty(localParams)) { if (!StringUtils.isEmpty(localParams)) {
localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams);
List<Property> localParamsList = JSONUtils.toList(localParams, Property.class); List<Property> localParamsList = JSONUtils.toList(localParams, Property.class);

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java

@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.api.service.QueueService;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper; import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -271,7 +271,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
* @return true if the queue not exists, otherwise return false * @return true if the queue not exists, otherwise return false
*/ */
private boolean checkQueueExist(String queue) { private boolean checkQueueExist(String queue) {
return BooleanUtils.isTrue(queueMapper.existQueue(queue, null)); return queueMapper.existQueue(queue, null) == Boolean.TRUE;
} }
/** /**
@ -282,7 +282,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
* @return true if the queue name not exists, otherwise return false * @return true if the queue name not exists, otherwise return false
*/ */
private boolean checkQueueNameExist(String queueName) { private boolean checkQueueNameExist(String queueName) {
return BooleanUtils.isTrue(queueMapper.existQueue(null, queueName)); return queueMapper.existQueue(null, queueName) == Boolean.TRUE;
} }
/** /**
@ -294,7 +294,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
* @return true if need to update user * @return true if need to update user
*/ */
private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) { private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) {
return !oldQueue.equals(newQueue) && BooleanUtils.isTrue(userMapper.existUser(oldQueue)); return !oldQueue.equals(newQueue) && userMapper.existUser(oldQueue) == Boolean.TRUE;
} }
} }

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@ -34,13 +34,11 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser; import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.Tenant;
@ -55,6 +53,7 @@ import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
import org.apache.commons.beanutils.BeanMap; import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.text.MessageFormat; import java.text.MessageFormat;
@ -253,7 +252,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
*/ */
private boolean checkResourceExists(String fullName, int userId, int type) { private boolean checkResourceExists(String fullName, int userId, int type) {
Boolean existResource = resourcesMapper.existResource(fullName, userId, type); Boolean existResource = resourcesMapper.existResource(fullName, userId, type);
return BooleanUtils.isTrue(existResource); return existResource == Boolean.TRUE;
} }
/** /**

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java

@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.Schedule;
@ -48,6 +47,8 @@ import org.apache.dolphinscheduler.service.quartz.ProcessScheduleJob;
import org.apache.dolphinscheduler.service.quartz.QuartzExecutors; import org.apache.dolphinscheduler.service.quartz.QuartzExecutors;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.commons.lang.StringUtils;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -250,7 +251,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
Date now = new Date(); Date now = new Date();
// updateProcessInstance param // updateProcessInstance param
if (StringUtils.isNotEmpty(scheduleExpression)) { if (!StringUtils.isEmpty(scheduleExpression)) {
ScheduleParam scheduleParam = JSONUtils.parseObject(scheduleExpression, ScheduleParam.class); ScheduleParam scheduleParam = JSONUtils.parseObject(scheduleExpression, ScheduleParam.class);
if (DateUtils.differSec(scheduleParam.getStartTime(), scheduleParam.getEndTime()) == 0) { if (DateUtils.differSec(scheduleParam.getStartTime(), scheduleParam.getEndTime()) == 0) {
logger.warn("The start time must not be the same as the end"); logger.warn("The start time must not be the same as the end");

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java

@ -21,11 +21,12 @@ import org.apache.dolphinscheduler.api.controller.BaseController;
import org.apache.dolphinscheduler.api.service.SessionService; import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Session; import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper; import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@ -40,6 +39,8 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java

@ -23,11 +23,9 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.Tenant;
@ -37,6 +35,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -202,7 +202,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
Date now = new Date(); Date now = new Date();
if (StringUtils.isNotEmpty(tenantCode)) { if (!StringUtils.isEmpty(tenantCode)) {
tenant.setTenantCode(tenantCode); tenant.setTenantCode(tenantCode);
} }
@ -322,6 +322,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
*/ */
private boolean checkTenantExists(String tenantCode) { private boolean checkTenantExists(String tenantCode) {
Boolean existTenant = tenantMapper.existTenant(tenantCode); Boolean existTenant = tenantMapper.existTenant(tenantCode);
return BooleanUtils.isTrue(existTenant); return existTenant == Boolean.TRUE;
} }
} }

9
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
@ -32,6 +31,8 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -113,10 +114,10 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setUserId(loginUser.getId()); udf.setUserId(loginUser.getId());
udf.setFuncName(funcName); udf.setFuncName(funcName);
udf.setClassName(className); udf.setClassName(className);
if (StringUtils.isNotEmpty(argTypes)) { if (!StringUtils.isEmpty(argTypes)) {
udf.setArgTypes(argTypes); udf.setArgTypes(argTypes);
} }
if (StringUtils.isNotEmpty(database)) { if (!StringUtils.isEmpty(database)) {
udf.setDatabase(database); udf.setDatabase(database);
} }
udf.setDescription(desc); udf.setDescription(desc);
@ -221,7 +222,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setFuncName(funcName); udf.setFuncName(funcName);
udf.setClassName(className); udf.setClassName(className);
udf.setArgTypes(argTypes); udf.setArgTypes(argTypes);
if (StringUtils.isNotEmpty(database)) { if (!StringUtils.isEmpty(database)) {
udf.setDatabase(database); udf.setDatabase(database);
} }
udf.setDescription(desc); udf.setDescription(desc);

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -33,7 +33,6 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils; import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.DatasourceUser; import org.apache.dolphinscheduler.dao.entity.DatasourceUser;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
@ -56,6 +55,8 @@ import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java

@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessLineage; import org.apache.dolphinscheduler.dao.entity.ProcessLineage;
import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
@ -28,6 +27,8 @@ import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.apache.commons.lang.StringUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java

@ -26,13 +26,14 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
@ -188,7 +189,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
if (CollectionUtils.isNotEmpty(workerGroups)) { if (CollectionUtils.isNotEmpty(workerGroups)) {
List<WorkerGroup> searchValDataList = new ArrayList<>(); List<WorkerGroup> searchValDataList = new ArrayList<>();
if (StringUtils.isNotEmpty(searchVal)) { if (!StringUtils.isEmpty(searchVal)) {
for (WorkerGroup workerGroup : workerGroups) { for (WorkerGroup workerGroup : workerGroups) {
if (workerGroup.getName().contains(searchVal)) { if (workerGroup.getName().contains(searchVal)) {
searchValDataList.add(workerGroup); searchValDataList.add(workerGroup);

23
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java

@ -22,10 +22,11 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -73,7 +74,7 @@ public class CheckUtils {
*/ */
public static Map<String, Object> checkDesc(String desc) { public static Map<String, Object> checkDesc(String desc) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
if (StringUtils.isNotEmpty(desc) && desc.length() > 200) { if (!StringUtils.isEmpty(desc) && desc.length() > 200) {
result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR);
result.put(Constants.MSG, result.put(Constants.MSG,
MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "desc length")); MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "desc length"));
@ -90,7 +91,7 @@ public class CheckUtils {
* @return true if other parameters are valid, otherwise return false * @return true if other parameters are valid, otherwise return false
*/ */
public static boolean checkOtherParams(String otherParams) { public static boolean checkOtherParams(String otherParams) {
return StringUtils.isNotEmpty(otherParams) && !JSONUtils.checkJsonValid(otherParams); return !StringUtils.isEmpty(otherParams) && !JSONUtils.checkJsonValid(otherParams);
} }
/** /**
@ -100,7 +101,7 @@ public class CheckUtils {
* @return true if password regex valid, otherwise return false * @return true if password regex valid, otherwise return false
*/ */
public static boolean checkPassword(String password) { public static boolean checkPassword(String password) {
return StringUtils.isNotEmpty(password) && password.length() >= 2 && password.length() <= 20; return !StringUtils.isEmpty(password) && password.length() >= 2 && password.length() <= 20;
} }
/** /**
@ -156,20 +157,6 @@ public class CheckUtils {
return false; return false;
} }
/**
* check params
*
* @param userName user name
* @param password password
* @param email email
* @param phone phone
* @return true if user parameters are valid, other return false
*/
public static boolean checkUserParams(String userName, String password, String email, String phone) {
return CheckUtils.checkUserName(userName) && CheckUtils.checkEmail(email) && CheckUtils.checkPassword(password)
&& CheckUtils.checkPhone(phone);
}
/** /**
* regex check * regex check
* *

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java

@ -60,40 +60,6 @@ public class RegistryCenterUtils {
return registryClient.getServerList(NodeType.WORKER); return registryClient.getServerList(NodeType.WORKER);
} }
private static List<ZookeeperRecord> zookeeperInfoList(String zookeeperServers) {
List<ZookeeperRecord> list = new ArrayList<>(5);
/*
if (StringUtils.isNotBlank(zookeeperServers)) {
String[] zookeeperServersArray = zookeeperServers.split(",");
for (String zookeeperServer : zookeeperServersArray) {
ZooKeeperState state = new ZooKeeperState(zookeeperServer);
boolean ok = state.ruok();
if (ok) {
state.getZookeeperInfo();
}
int connections = state.getConnections();
int watches = state.getWatches();
long sent = state.getSent();
long received = state.getReceived();
String mode = state.getMode();
float minLatency = state.getMinLatency();
float avgLatency = state.getAvgLatency();
float maxLatency = state.getMaxLatency();
int nodeCount = state.getNodeCount();
int status = ok ? 1 : 0;
Date date = new Date();
ZookeeperRecord zookeeperRecord = new ZookeeperRecord(zookeeperServer,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date);
list.add(zookeeperRecord);
}
}*/
return list;
}
public static Map<String, String> getServerMaps(NodeType nodeType, boolean hostOnly) { public static Map<String, String> getServerMaps(NodeType nodeType, boolean hostOnly) {
return registryClient.getServerMaps(nodeType, hostOnly); return registryClient.getServerMaps(nodeType, hostOnly);
} }

226
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java

@ -1,226 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.utils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Scanner;
/**
* zookeeper state monitor
*
*/
public class ZooKeeperState {
private static final Logger logger = LoggerFactory.getLogger(ZooKeeperState.class);
private final String host;
private final int port;
private float minLatency = -1, avgLatency = -1, maxLatency = -1;
private long received = -1;
private long sent = -1;
private int outStanding = -1;
private long zxid = -1;
private String mode = null;
private int nodeCount = -1;
private int watches = -1;
private int connections = -1;
public ZooKeeperState(String connectionString) {
String host = connectionString.substring(0,
connectionString.indexOf(':'));
int port = Integer.parseInt(connectionString.substring(connectionString
.indexOf(':') + 1));
this.host = host;
this.port = port;
}
public void getZookeeperInfo() {
String content = cmd("srvr");
if (StringUtils.isNotBlank(content)) {
try (Scanner scannerForStat = new Scanner(content)) {
while (scannerForStat.hasNext()) {
String line = scannerForStat.nextLine();
if (line.startsWith("Latency min/avg/max:")) {
String[] latencys = getStringValueFromLine(line).split("/");
minLatency = Float.parseFloat(latencys[0]);
avgLatency = Float.parseFloat(latencys[1]);
maxLatency = Float.parseFloat(latencys[2]);
} else if (line.startsWith("Received:")) {
received = Long.parseLong(getStringValueFromLine(line));
} else if (line.startsWith("Sent:")) {
sent = Long.parseLong(getStringValueFromLine(line));
} else if (line.startsWith("Outstanding:")) {
outStanding = Integer.parseInt(getStringValueFromLine(line));
} else if (line.startsWith("Zxid:")) {
zxid = Long.parseLong(getStringValueFromLine(line).substring(2), 16);
} else if (line.startsWith("Mode:")) {
mode = getStringValueFromLine(line);
} else if (line.startsWith("Node count:")) {
nodeCount = Integer.parseInt(getStringValueFromLine(line));
}
}
}
}
String wchsText = cmd("wchs");
if (StringUtils.isNotBlank(wchsText)) {
try (Scanner scannerForWchs = new Scanner(wchsText)) {
while (scannerForWchs.hasNext()) {
String line = scannerForWchs.nextLine();
if (line.startsWith("Total watches:")) {
watches = Integer.parseInt(getStringValueFromLine(line));
}
}
}
}
String consText = cmd("cons");
if (StringUtils.isNotBlank(consText)) {
Scanner scannerForCons = new Scanner(consText);
if (StringUtils.isNotBlank(consText)) {
connections = 0;
}
while (scannerForCons.hasNext()) {
@SuppressWarnings("unused")
String line = scannerForCons.nextLine();
++connections;
}
scannerForCons.close();
}
}
public boolean ruok() {
return "imok\n".equals(cmd("ruok"));
}
private String getStringValueFromLine(String line) {
return line.substring(line.indexOf(":") + 1, line.length()).replaceAll(
" ", "").trim();
}
private class SendThread extends Thread {
private String cmd;
private String ret = "";
public SendThread(String cmd) {
this.cmd = cmd;
}
@Override
public void run() {
try {
ret = FourLetterWordMain.send4LetterWord(host, port, cmd);
} catch (Exception e) {
logger.error(e.getMessage(),e);
return;
}
}
}
private String cmd(String cmd) {
final int waitTimeout = 5;
SendThread sendThread = new SendThread(cmd);
sendThread.setName("FourLetterCmd:" + cmd);
sendThread.start();
try {
sendThread.join(waitTimeout * 1000L);
return sendThread.ret;
} catch (InterruptedException e) {
logger.error("send " + cmd + " to server " + host + ":" + port + " failed!", e);
Thread.currentThread().interrupt();
}
return "";
}
public Logger getLogger() {
return logger;
}
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public float getMinLatency() {
return minLatency;
}
public float getAvgLatency() {
return avgLatency;
}
public float getMaxLatency() {
return maxLatency;
}
public long getReceived() {
return received;
}
public long getSent() {
return sent;
}
public int getOutStanding() {
return outStanding;
}
public long getZxid() {
return zxid;
}
public String getMode() {
return mode;
}
public int getNodeCount() {
return nodeCount;
}
public int getWatches() {
return watches;
}
public int getConnections() {
return connections;
}
@Override
public String toString() {
return "ZooKeeperState [host=" + host + ", port=" + port
+ ", minLatency=" + minLatency + ", avgLatency=" + avgLatency
+ ", maxLatency=" + maxLatency + ", received=" + received
+ ", sent=" + sent + ", outStanding=" + outStanding + ", zxid="
+ zxid + ", mode=" + mode + ", nodeCount=" + nodeCount
+ ", watches=" + watches + ", connections="
+ connections + "]";
}
}

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java

@ -23,10 +23,11 @@ import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils; import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.service.registry.RegistryClient; import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -97,7 +98,7 @@ public class AbstractControllerTest {
String session = sessionService.createSession(loginUser, "127.0.0.1"); String session = sessionService.createSession(loginUser, "127.0.0.1");
sessionId = session; sessionId = session;
Assert.assertTrue(StringUtils.isNotEmpty(session)); Assert.assertTrue(!StringUtils.isEmpty(session));
} }
public Map<String, Object> success() { public Map<String, Object> success() {

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
@ -47,6 +46,8 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java

@ -21,11 +21,12 @@ import org.apache.dolphinscheduler.api.service.impl.SessionServiceImpl;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Session; import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper; import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
@ -109,7 +110,7 @@ public class SessionServiceTest {
Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions()); Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions());
String sessionId = sessionService.createSession(user, ip); String sessionId = sessionService.createSession(user, ip);
logger.info("createSessionId is " + sessionId); logger.info("createSessionId is " + sessionId);
Assert.assertTrue(StringUtils.isNotEmpty(sessionId)); Assert.assertTrue(!StringUtils.isEmpty(sessionId));
} }
/** /**

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java

@ -106,7 +106,7 @@ public class FileUtilsTest {
@Test @Test
public void testFile2String() throws IOException { public void testFile2String() throws IOException {
String content = "123"; String content = "123";
org.apache.dolphinscheduler.common.utils.FileUtils.writeStringToFile(new File("/tmp/task.json"),content); org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/task.json"),content);
File file = new File("/tmp/task.json"); File file = new File("/tmp/task.json");
FileInputStream fileInputStream = new FileInputStream("/tmp/task.json"); FileInputStream fileInputStream = new FileInputStream("/tmp/task.json");

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -1095,8 +1096,8 @@ public final class Constants {
/** /**
* docker & kubernetes * docker & kubernetes
*/ */
public static final boolean DOCKER_MODE = StringUtils.isNotEmpty(System.getenv("DOCKER")); public static final boolean DOCKER_MODE = !StringUtils.isEmpty(System.getenv("DOCKER"));
public static final boolean KUBERNETES_MODE = StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_PORT")); public static final boolean KUBERNETES_MODE = !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
/** /**
* task parameter keys * task parameter keys

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java

@ -24,9 +24,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -83,7 +83,7 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam;
String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); String jdbcUrl = clickhouseConnectionParam.getJdbcUrl();
if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { if (!StringUtils.isEmpty(clickhouseConnectionParam.getOther())) {
jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther());
} }
return jdbcUrl; return jdbcUrl;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -84,7 +84,7 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor {
@Override @Override
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { if (!StringUtils.isEmpty(db2ConnectionParam.getOther())) {
return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther());
} }
return db2ConnectionParam.getJdbcUrl(); return db2ConnectionParam.getJdbcUrl();

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java

@ -26,9 +26,9 @@ import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HiveConfUtils; import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.sql.Connection; import java.sql.Connection;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -102,7 +102,7 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam;
String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); String jdbcUrl = mysqlConnectionParam.getJdbcUrl();
if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { if (!StringUtils.isEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS);
} }
return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); return String.format("%s?%s", jdbcUrl, APPEND_PARAMS);

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java

@ -26,9 +26,9 @@ import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -99,7 +99,7 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor {
@Override @Override
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { if (!StringUtils.isEmpty(oracleConnectionParam.getOther())) {
return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther());
} }
return oracleConnectionParam.getJdbcUrl(); return oracleConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -84,7 +84,7 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor {
@Override @Override
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { if (!StringUtils.isEmpty(postgreSqlConnectionParam.getOther())) {
return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther());
} }
return postgreSqlConnectionParam.getJdbcUrl(); return postgreSqlConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -86,7 +86,7 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor {
@Override @Override
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { if (!StringUtils.isEmpty(prestoConnectionParam.getOther())) {
return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther());
} }
return prestoConnectionParam.getJdbcUrl(); return prestoConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.sql.Connection; import java.sql.Connection;
@ -110,7 +110,7 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor {
@Override @Override
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { if (!StringUtils.isEmpty(sparkConnectionParam.getOther())) {
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther());
} }
return sparkConnectionParam.getJdbcUrl(); return sparkConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
@ -82,7 +82,7 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) { public String getJdbcUrl(ConnectionParam connectionParam) {
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { if (!StringUtils.isEmpty(sqlServerConnectionParam.getOther())) {
return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther());
} }
return sqlServerConnectionParam.getJdbcUrl(); return sqlServerConnectionParam.getJdbcUrl();

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java

@ -24,7 +24,8 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -245,7 +246,7 @@ public class TaskNode {
} }
public Boolean isForbidden() { public Boolean isForbidden() {
return (StringUtils.isNotEmpty(this.runFlag) return (!StringUtils.isEmpty(this.runFlag)
&& this.runFlag.equals(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN)); && this.runFlag.equals(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN));
} }
@ -363,7 +364,7 @@ public class TaskNode {
* @return task time out parameter * @return task time out parameter
*/ */
public TaskTimeoutParameter getTaskTimeoutParameter() { public TaskTimeoutParameter getTaskTimeoutParameter() {
if (StringUtils.isNotEmpty(this.getTimeout())) { if (!StringUtils.isEmpty(this.getTimeout())) {
String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name()); String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name()); String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name());
return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class); return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class);

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java

@ -21,7 +21,8 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -168,7 +169,7 @@ public abstract class AbstractParameters implements IParameters {
String[] formatResult = result.split("\\$VarPool\\$"); String[] formatResult = result.split("\\$VarPool\\$");
Map<String, String> format = new HashMap<>(); Map<String, String> format = new HashMap<>();
for (String info : formatResult) { for (String info : formatResult) {
if (StringUtils.isNotEmpty(info) && info.contains("=")) { if (!StringUtils.isEmpty(info) && info.contains("=")) {
String[] keyValue = info.split("="); String[] keyValue = info.split("=");
format.put(keyValue[0], keyValue[1]); format.put(keyValue[0], keyValue[1]);
} }

9
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java

@ -20,7 +20,8 @@ package org.apache.dolphinscheduler.common.task.datax;
import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -217,10 +218,10 @@ public class DataxParameters extends AbstractParameters {
if (customConfig == Flag.NO.ordinal()) { if (customConfig == Flag.NO.ordinal()) {
return dataSource != 0 return dataSource != 0
&& dataTarget != 0 && dataTarget != 0
&& StringUtils.isNotEmpty(sql) && !StringUtils.isEmpty(sql)
&& StringUtils.isNotEmpty(targetTable); && !StringUtils.isEmpty(targetTable);
} else { } else {
return StringUtils.isNotEmpty(json); return !StringUtils.isEmpty(json);
} }
} }

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java

@ -22,7 +22,8 @@ import org.apache.dolphinscheduler.common.enums.HttpMethod;
import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.HttpProperty;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -71,7 +72,7 @@ public class HttpParameters extends AbstractParameters {
@Override @Override
public boolean checkParameters() { public boolean checkParameters() {
return StringUtils.isNotEmpty(url); return !StringUtils.isEmpty(url);
} }
@Override @Override

118
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java

@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.common.task.procedure;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -29,62 +30,61 @@ import java.util.List;
*/ */
public class ProcedureParameters extends AbstractParameters { public class ProcedureParameters extends AbstractParameters {
/** /**
* data source typeeg MYSQL, POSTGRES, HIVE ... * data source typeeg MYSQL, POSTGRES, HIVE ...
*/ */
private String type; private String type;
/** /**
* data source id * data source id
*/ */
private int datasource; private int datasource;
/** /**
* procedure name * procedure name
*/ */
private String method; private String method;
public String getType() {
public String getType() { return type;
return type; }
}
public void setType(String type) {
public void setType(String type) { this.type = type;
this.type = type; }
}
public int getDatasource() {
public int getDatasource() { return datasource;
return datasource; }
}
public void setDatasource(int datasource) {
public void setDatasource(int datasource) { this.datasource = datasource;
this.datasource = datasource; }
}
public String getMethod() {
public String getMethod() { return method;
return method; }
}
public void setMethod(String method) {
public void setMethod(String method) { this.method = method;
this.method = method; }
}
@Override
@Override public boolean checkParameters() {
public boolean checkParameters() { return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(method);
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(method); }
}
@Override
@Override public List<ResourceInfo> getResourceFilesList() {
public List<ResourceInfo> getResourceFilesList() { return new ArrayList<>();
return new ArrayList<>(); }
}
@Override
@Override public String toString() {
public String toString() { return "ProcessdureParam{"
return "ProcessdureParam{" + + "type='" + type + '\''
"type='" + type + '\'' + + ", datasource=" + datasource
", datasource=" + datasource + + ", method='" + method + '\''
", method='" + method + '\'' + + '}';
'}'; }
}
} }

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java

@ -23,7 +23,8 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -217,7 +218,7 @@ public class SqlParameters extends AbstractParameters {
@Override @Override
public boolean checkParameters() { public boolean checkParameters() {
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(sql); return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(sql);
} }
@Override @Override

25
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java

@ -20,7 +20,8 @@ import org.apache.dolphinscheduler.common.enums.SqoopJobType;
import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -28,7 +29,7 @@ import java.util.List;
/** /**
* sqoop parameters * sqoop parameters
*/ */
public class SqoopParameters extends AbstractParameters { public class SqoopParameters extends AbstractParameters {
/** /**
* sqoop job type: * sqoop job type:
@ -180,17 +181,17 @@ public class SqoopParameters extends AbstractParameters {
} }
if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) { if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isEmpty(customShell) && sqoopParamsCheck = StringUtils.isEmpty(customShell)
StringUtils.isNotEmpty(modelType) && && !StringUtils.isEmpty(modelType)
StringUtils.isNotEmpty(jobName) && && !StringUtils.isEmpty(jobName)
concurrency != 0 && && concurrency != 0
StringUtils.isNotEmpty(sourceType) && && !StringUtils.isEmpty(sourceType)
StringUtils.isNotEmpty(targetType) && && !StringUtils.isEmpty(targetType)
StringUtils.isNotEmpty(sourceParams) && && !StringUtils.isEmpty(sourceParams)
StringUtils.isNotEmpty(targetParams); && !StringUtils.isEmpty(targetParams);
} else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) { } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isNotEmpty(customShell) && sqoopParamsCheck = !StringUtils.isEmpty(customShell)
StringUtils.isEmpty(jobName); && StringUtils.isEmpty(jobName);
} }
return sqoopParamsCheck; return sqoopParamsCheck;

33
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
public class BooleanUtils {
public static boolean isTrue(Boolean bool) {
if (bool == null) {
return false;
} else {
return bool;
}
}
public static boolean isNotTrue(Boolean bool) {
return !isTrue(bool);
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap; import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;

99
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.lang.StringUtils;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId; import java.time.ZoneId;
@ -26,16 +28,12 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.Objects;
import java.util.TimeZone; import java.util.TimeZone;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** public final class DateUtils {
* date utils
*/
public class DateUtils {
static final long C0 = 1L; static final long C0 = 1L;
static final long C1 = C0 * 1000L; static final long C1 = C0 * 1000L;
@ -45,36 +43,12 @@ public class DateUtils {
static final long C5 = C4 * 60L; static final long C5 = C4 * 60L;
static final long C6 = C5 * 24L; static final long C6 = C5 * 24L;
/**
* a default datetime formatter for the timestamp
*/
private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() { private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils"); throw new UnsupportedOperationException("Construct DateUtils");
} }
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @return string formatted as yyyy-MM-dd HH:mm:ss
*/
public static String formatTimeStamp(long timeMillis) {
return formatTimeStamp(timeMillis, DEFAULT_DATETIME_FORMATTER);
}
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @param dateTimeFormatter expect formatter, like yyyy-MM-dd HH:mm:ss
* @return formatted string
*/
public static String formatTimeStamp(long timeMillis, DateTimeFormatter dateTimeFormatter) {
Objects.requireNonNull(dateTimeFormatter);
return dateTimeFormatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeMillis),
ZoneId.systemDefault()));
}
/** /**
* date to local datetime * date to local datetime
* *
@ -199,28 +173,6 @@ public class DateUtils {
return Math.abs(d1.getTime() - d2.getTime()); return Math.abs(d1.getTime() - d2.getTime());
} }
/**
* get hours between two dates
*
* @param d1 date1
* @param d2 date2
* @return differ hours
*/
public static long diffHours(Date d1, Date d2) {
return (long) Math.ceil(diffMin(d1, d2) / 60.0);
}
/**
* get minutes between two dates
*
* @param d1 date1
* @param d2 date2
* @return differ minutes
*/
public static long diffMin(Date d1, Date d2) {
return (long) Math.ceil(differSec(d1, d2) / 60.0);
}
/** /**
* get the date of the specified date in the days before and after * get the date of the specified date in the days before and after
* *
@ -483,38 +435,6 @@ public class DateUtils {
Constants.YYYY_MM_DD_HH_MM_SS); Constants.YYYY_MM_DD_HH_MM_SS);
} }
public static Date addYears(Date date, int amount) {
return add(date, 1, amount);
}
public static Date addMonths(Date date, int amount) {
return add(date, 2, amount);
}
public static Date addWeeks(Date date, int amount) {
return add(date, 3, amount);
}
public static Date addDays(Date date, int amount) {
return add(date, 5, amount);
}
public static Date addHours(Date date, int amount) {
return add(date, 11, amount);
}
public static Date addMinutes(Date date, int amount) {
return add(date, 12, amount);
}
public static Date addSeconds(Date date, int amount) {
return add(date, 13, amount);
}
public static Date addMilliseconds(Date date, int amount) {
return add(date, 14, amount);
}
/** /**
* get date * get date
* *
@ -590,19 +510,6 @@ public class DateUtils {
* Time unit representing one thousandth of a second * Time unit representing one thousandth of a second
*/ */
public static class MILLISECONDS { public static class MILLISECONDS {
public static long toSeconds(long d) {
return d / (C3 / C2);
}
public static long toMinutes(long d) {
return d / (C4 / C2);
}
public static long toHours(long d) {
return d / (C5 / C2);
}
public static long toDays(long d) { public static long toDays(long d) {
return d / (C6 / C2); return d / (C6 / C2);
} }

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
/** /**
* encryption utils * encryption utils

185
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@ -22,8 +22,8 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE; import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.BufferedWriter; import java.io.BufferedWriter;
@ -32,13 +32,9 @@ import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.StringReader; import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional; import java.util.Optional;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -68,7 +64,7 @@ public class FileUtils {
public static String suffix(String filename) { public static String suffix(String filename) {
String fileSuffix = ""; String fileSuffix = "";
if (StringUtils.isNotEmpty(filename)) { if (!StringUtils.isEmpty(filename)) {
int lastIndex = filename.lastIndexOf('.'); int lastIndex = filename.lastIndexOf('.');
if (lastIndex > 0) { if (lastIndex > 0) {
fileSuffix = filename.substring(lastIndex + 1); fileSuffix = filename.substring(lastIndex + 1);
@ -168,7 +164,6 @@ public class FileUtils {
* @return true if write success * @return true if write success
*/ */
public static boolean writeContent2File(String content, String filePath) { public static boolean writeContent2File(String content, String filePath) {
boolean flag = true;
BufferedReader bufferedReader = null; BufferedReader bufferedReader = null;
BufferedWriter bufferedWriter = null; BufferedWriter bufferedWriter = null;
try { try {
@ -189,184 +184,12 @@ public class FileUtils {
bufferedWriter.close(); bufferedWriter.close();
} catch (IOException e) { } catch (IOException e) {
FileUtils.logger.error(e.getMessage(), e); FileUtils.logger.error(e.getMessage(), e);
flag = false; return false;
return flag;
} finally { } finally {
IOUtils.closeQuietly(bufferedWriter); IOUtils.closeQuietly(bufferedWriter);
IOUtils.closeQuietly(bufferedReader); IOUtils.closeQuietly(bufferedReader);
} }
return flag; return true;
}
/**
* Writes a String to a file creating the file if it does not exist.
* <p>
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
* @since 2.4
*/
public static void writeStringToFile(File file, String data, Charset encoding) throws IOException {
writeStringToFile(file, data, encoding, false);
}
/**
* Writes a String to a file creating the file if it does not exist.
* <p>
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
*/
public static void writeStringToFile(File file, String data, String encoding) throws IOException {
writeStringToFile(file, data, encoding, false);
}
/**
* Writes a String to a file creating the file if it does not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @since 2.3
*/
public static void writeStringToFile(File file, String data, Charset encoding, boolean append) throws IOException {
OutputStream out = null;
try {
out = openOutputStream(file, append);
IOUtils.write(data, out, encoding);
out.close(); // don't swallow close Exception if copy completes normally
} finally {
IOUtils.closeQuietly(out);
}
}
/**
* Writes a String to a file creating the file if it does not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* supported by the VM
* @since 2.1
*/
public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException {
writeStringToFile(file, data, Charsets.toCharset(encoding), append);
}
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
* @param file the file to write
* @param data the content to write to the file
* @throws IOException in case of an I/O error
*/
public static void writeStringToFile(File file, String data) throws IOException {
writeStringToFile(file, data, Charset.defaultCharset(), false);
}
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
* @param file the file to write
* @param data the content to write to the file
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @since 2.1
*/
public static void writeStringToFile(File file, String data, boolean append) throws IOException {
writeStringToFile(file, data, Charset.defaultCharset(), append);
}
/**
* Opens a {@link FileOutputStream} for the specified file, checking and
* creating the parent directory if it does not exist.
* <p>
* At the end of the method either the stream will be successfully opened,
* or an exception will have been thrown.
* <p>
* The parent directory will be created if it does not exist.
* The file will be created if it does not exist.
* An exception is thrown if the file object exists but is a directory.
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
* @param file the file to open for output, must not be {@code null}
* @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to
* @throws IOException if a parent directory needs creating but that fails
* @since 1.3
*/
public static FileOutputStream openOutputStream(File file) throws IOException {
return openOutputStream(file, false);
}
/**
* Opens a {@link FileOutputStream} for the specified file, checking and
* creating the parent directory if it does not exist.
* <p>
* At the end of the method either the stream will be successfully opened,
* or an exception will have been thrown.
* <p>
* The parent directory will be created if it does not exist.
* The file will be created if it does not exist.
* An exception is thrown if the file object exists but is a directory.
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
* @param file the file to open for output, must not be {@code null}
* @param append if {@code true}, then bytes will be added to the
* end of the file rather than overwriting
* @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to
* @throws IOException if a parent directory needs creating but that fails
* @since 2.1
*/
public static FileOutputStream openOutputStream(File file, boolean append) throws IOException {
if (file.exists()) {
if (file.isDirectory()) {
throw new IOException("File '" + file + "' exists but is a directory");
}
if (!file.canWrite()) {
throw new IOException("File '" + file + "' cannot be written to");
}
} else {
File parent = file.getParentFile();
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
throw new IOException("Directory '" + parent + "' could not be created");
}
}
return new FileOutputStream(file, append);
}
/**
* deletes a directory recursively
*
* @param dir directory
* @throws IOException in case deletion is unsuccessful
*/
public static void deleteDir(String dir) throws IOException {
org.apache.commons.io.FileUtils.deleteDirectory(new File(dir));
} }
/** /**

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.exception.BaseException; import org.apache.dolphinscheduler.common.exception.BaseException;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java

@ -24,6 +24,8 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
import org.apache.commons.lang.StringUtils;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.shell.ShellExecutor; import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
@ -206,7 +208,7 @@ public class OSUtils {
*/ */
private static List<String> getUserListFromMac() throws IOException { private static List<String> getUserListFromMac() throws IOException {
String result = exeCmd("dscl . list /users"); String result = exeCmd("dscl . list /users");
if (StringUtils.isNotEmpty(result)) { if (!StringUtils.isEmpty(result)) {
return Arrays.asList(result.split("\n")); return Arrays.asList(result.split("\n"));
} }
@ -392,7 +394,7 @@ public class OSUtils {
} }
} else { } else {
String result = exeCmd("groups"); String result = exeCmd("groups");
if (StringUtils.isNotEmpty(result)) { if (!StringUtils.isEmpty(result)) {
String[] groupInfo = result.split(" "); String[] groupInfo = result.split(" ");
return groupInfo[0]; return groupInfo[0];
} }

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -25,6 +25,8 @@ import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
@ -99,7 +101,7 @@ public class ParameterUtils {
String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME); String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
Date cronTime = null; Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) { if (!StringUtils.isEmpty(cronTimeStr)) {
cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME); cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
} else { } else {
@ -209,7 +211,7 @@ public class ParameterUtils {
*/ */
public static String handleEscapes(String inputString) { public static String handleEscapes(String inputString) {
if (StringUtils.isNotEmpty(inputString)) { if (!StringUtils.isEmpty(inputString)) {
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
} }
return inputString; return inputString;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java

@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
import org.apache.commons.lang.StringUtils;
/** /**
* heartbeat for ZK reigster res info * heartbeat for ZK reigster res info
*/ */
@ -113,7 +115,7 @@ public class ResInfo {
* @return heartbeat info is valid * @return heartbeat info is valid
*/ */
public static boolean isValidHeartbeatForRegistryInfo(String heartBeatInfo) { public static boolean isValidHeartbeatForRegistryInfo(String heartBeatInfo) {
if (StringUtils.isNotEmpty(heartBeatInfo)) { if (!StringUtils.isEmpty(heartBeatInfo)) {
String[] parts = heartBeatInfo.split(Constants.COMMA); String[] parts = heartBeatInfo.split(Constants.COMMA);
return parts.length == Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH return parts.length == Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH
|| parts.length == Constants.HEARTBEAT_WITH_WEIGHT_FOR_ZOOKEEPER_INFO_LENGTH; || parts.length == Constants.HEARTBEAT_WITH_WEIGHT_FOR_ZOOKEEPER_INFO_LENGTH;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;

310
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java

@ -1,12 +1,12 @@
/** /*
* Copyright 2004-2020 the original author or authors. * Copyright 2004-2020 the original author or authors.
* * <p>
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -32,113 +32,85 @@ import org.slf4j.LoggerFactory;
*/ */
public class ScriptRunner { public class ScriptRunner {
public static final Logger logger = LoggerFactory.getLogger(ScriptRunner.class); public static final Logger logger = LoggerFactory.getLogger(ScriptRunner.class);
private static final String DEFAULT_DELIMITER = ";"; private static final String DEFAULT_DELIMITER = ";";
private Connection connection; private final Connection connection;
private boolean stopOnError; private final boolean stopOnError;
private boolean autoCommit; private final boolean autoCommit;
private String delimiter = DEFAULT_DELIMITER; private String delimiter = DEFAULT_DELIMITER;
private boolean fullLineDelimiter = false; private boolean fullLineDelimiter = false;
public ScriptRunner(Connection connection, boolean autoCommit, boolean stopOnError) {
public ScriptRunner(Connection connection, boolean autoCommit, boolean stopOnError) { this.connection = connection;
this.connection = connection; this.autoCommit = autoCommit;
this.autoCommit = autoCommit; this.stopOnError = stopOnError;
this.stopOnError = stopOnError; }
}
public void setDelimiter(String delimiter, boolean fullLineDelimiter) {
this.delimiter = delimiter;
public void setDelimiter(String delimiter, boolean fullLineDelimiter) { this.fullLineDelimiter = fullLineDelimiter;
this.delimiter = delimiter; }
this.fullLineDelimiter = fullLineDelimiter;
} /**
* Runs an SQL script (read in using the Reader parameter)
/** *
* Runs an SQL script (read in using the Reader parameter) * @param reader - the source of the script
* * @throws IOException errors
* @param reader * @throws SQLException errors
* - the source of the script */
* @throws IOException errors public void runScript(Reader reader) throws IOException, SQLException {
* @throws SQLException errors try {
*/ boolean originalAutoCommit = connection.getAutoCommit();
public void runScript(Reader reader) throws IOException, SQLException { try {
try { if (originalAutoCommit != this.autoCommit) {
boolean originalAutoCommit = connection.getAutoCommit(); connection.setAutoCommit(this.autoCommit);
try { }
if (originalAutoCommit != this.autoCommit) { runScript(connection, reader);
connection.setAutoCommit(this.autoCommit); } finally {
} connection.setAutoCommit(originalAutoCommit);
runScript(connection, reader); }
} finally { } catch (IOException | SQLException e) {
connection.setAutoCommit(originalAutoCommit); throw e;
} } catch (Exception e) {
} catch (IOException | SQLException e) { throw new RuntimeException("Error running script. Cause: " + e, e);
throw e; }
} catch (Exception e) { }
throw new RuntimeException("Error running script. Cause: " + e, e);
} /**
} * Runs an SQL script (read in using the Reader parameter) using the connection
* passed in
public void runScript(Reader reader, String dbName) throws IOException, SQLException { *
try { * @param conn - the connection to use for the script
boolean originalAutoCommit = connection.getAutoCommit(); * @param reader - the source of the script
try { * @throws SQLException if any SQL errors occur
if (originalAutoCommit != this.autoCommit) { * @throws IOException if there is an error reading from the Reader
connection.setAutoCommit(this.autoCommit); */
} private void runScript(Connection conn, Reader reader) throws IOException, SQLException {
runScript(connection, reader, dbName); StringBuffer command = null;
} finally { try {
connection.setAutoCommit(originalAutoCommit); LineNumberReader lineReader = new LineNumberReader(reader);
} String line;
} catch (IOException | SQLException e) { while ((line = lineReader.readLine()) != null) {
throw e; if (command == null) {
} catch (Exception e) { command = new StringBuffer();
throw new RuntimeException("Error running script. Cause: " + e, e); }
} String trimmedLine = line.trim();
} if (trimmedLine.startsWith("--")) {
logger.info(trimmedLine);
/** } else if (trimmedLine.length() < 1 || trimmedLine.startsWith("//")) {
* Runs an SQL script (read in using the Reader parameter) using the connection // Do nothing
* passed in } else if (trimmedLine.startsWith("delimiter")) {
* String newDelimiter = trimmedLine.split(" ")[1];
* @param conn this.setDelimiter(newDelimiter, fullLineDelimiter);
* - the connection to use for the script
* @param reader } else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter())
* - the source of the script || fullLineDelimiter && trimmedLine.equals(getDelimiter())) {
* @throws SQLException command.append(line, 0, line.lastIndexOf(getDelimiter()));
* if any SQL errors occur command.append(" ");
* @throws IOException
* if there is an error reading from the Reader
*/
private void runScript(Connection conn, Reader reader) throws IOException, SQLException {
StringBuffer command = null;
try {
LineNumberReader lineReader = new LineNumberReader(reader);
String line = null;
while ((line = lineReader.readLine()) != null) {
if (command == null) {
command = new StringBuffer();
}
String trimmedLine = line.trim();
if (trimmedLine.startsWith("--")) {
logger.info(trimmedLine);
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("//")) {
// Do nothing
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("--")) {
// Do nothing
} else if (trimmedLine.startsWith("delimiter")) {
String newDelimiter = trimmedLine.split(" ")[1];
this.setDelimiter(newDelimiter, fullLineDelimiter);
} else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter())
|| fullLineDelimiter && trimmedLine.equals(getDelimiter())) {
command.append(line.substring(0, line.lastIndexOf(getDelimiter())));
command.append(" ");
logger.info("sql: {}", command); logger.info("sql: {}", command);
try (Statement statement = conn.createStatement()) { try (Statement statement = conn.createStatement()) {
@ -166,98 +138,26 @@ public class ScriptRunner {
throw e; throw e;
} }
command = null; command = null;
Thread.yield(); Thread.yield();
} else { } else {
command.append(line); command.append(line);
command.append(" "); command.append(" ");
} }
} }
} catch (SQLException e) { } catch (SQLException e) {
logger.error("Error executing: {}", command); logger.error("Error executing: {}", command);
throw e; throw e;
} catch (IOException e) { } catch (IOException e) {
e.fillInStackTrace(); e.fillInStackTrace();
logger.error("Error executing: {}", command); logger.error("Error executing: {}", command);
throw e; throw e;
} }
} }
private void runScript(Connection conn, Reader reader , String dbName) throws IOException, SQLException { private String getDelimiter() {
StringBuffer command = null; return delimiter;
String sql = ""; }
String appKey = dbName.substring(dbName.lastIndexOf("_")+1, dbName.length());
try {
LineNumberReader lineReader = new LineNumberReader(reader);
String line = null;
while ((line = lineReader.readLine()) != null) {
if (command == null) {
command = new StringBuffer();
}
String trimmedLine = line.trim();
if (trimmedLine.startsWith("--")) {
logger.info(trimmedLine);
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("//")) {
// Do nothing
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("--")) {
// Do nothing
} else if (trimmedLine.startsWith("delimiter")) {
String newDelimiter = trimmedLine.split(" ")[1];
this.setDelimiter(newDelimiter, fullLineDelimiter);
} else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter())
|| fullLineDelimiter && trimmedLine.equals(getDelimiter())) {
command.append(line.substring(0, line.lastIndexOf(getDelimiter())));
command.append(" ");
sql = command.toString().replaceAll("\\{\\{APPDB\\}\\}", dbName);
logger.info("sql : {}", sql);
try (Statement statement = conn.createStatement()) {
statement.execute(sql);
try (ResultSet rs = statement.getResultSet()) {
if (stopOnError && rs != null) {
ResultSetMetaData md = rs.getMetaData();
int cols = md.getColumnCount();
for (int i = 0; i < cols; i++) {
String name = md.getColumnLabel(i);
logger.info("{} \t", name);
}
logger.info("");
while (rs.next()) {
for (int i = 0; i < cols; i++) {
String value = rs.getString(i);
logger.info("{} \t", value);
}
logger.info("");
}
}
}
} catch (SQLException e) {
logger.error("SQLException", e);
throw e;
}
command = null;
Thread.yield();
} else {
command.append(line);
command.append(" ");
}
}
} catch (SQLException e) {
throw e;
} catch (IOException e) {
e.fillInStackTrace();
logger.error("Error executing: {}", sql);
throw e;
}
}
private String getDelimiter() {
return delimiter;
}
} }

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java

@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.lang.StringUtils;
/** /**
* sensitive log Util * sensitive log Util
*/ */
@ -34,7 +36,7 @@ public class SensitiveLogUtils {
*/ */
public static String maskDataSourcePwd(String dataSourcePwd) { public static String maskDataSourcePwd(String dataSourcePwd) {
if (StringUtils.isNotEmpty(dataSourcePwd)) { if (!StringUtils.isEmpty(dataSourcePwd)) {
dataSourcePwd = Constants.PASSWORD_DEFAULT; dataSourcePwd = Constants.PASSWORD_DEFAULT;
} }
return dataSourcePwd; return dataSourcePwd;

226
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -1,226 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.Collection;
import java.util.Iterator;
/**
* java.lang.String utils class
*/
public class StringUtils {
/**
* The empty String {@code ""}.
*/
public static final String EMPTY = "";
public static final int INDEX_NOT_FOUND = -1;
private StringUtils() {
throw new UnsupportedOperationException("Construct StringUtils");
}
/**
* <p>Checks if a CharSequence is empty ("") or null.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is empty or null
*/
public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
/**
* <p>Checks if a CharSequence is not empty ("") and not null.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is not empty and not null
*/
public static boolean isNotEmpty(final CharSequence cs) {
return !isEmpty(cs);
}
/**
* <p>Checks if a CharSequence is empty (""), null or whitespace only.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is null, empty or whitespace only
*/
public static boolean isBlank(final CharSequence cs) {
int strLen;
if (cs == null || (strLen = cs.length()) == 0) {
return true;
}
for (int i = 0; i < strLen; i++) {
if (!Character.isWhitespace(cs.charAt(i))) {
return false;
}
}
return true;
}
/**
* <p>Checks if a CharSequence is not empty (""), not null and not whitespace only.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is not empty and not null and not whitespace only
*/
public static boolean isNotBlank(final CharSequence cs) {
return !isBlank(cs);
}
/**
* <p>Replace all strings matching the regular expression \t \n \r with _</p>
*
* @param src the String , may be null
* @return the string that has been replaced
*/
public static String replaceNRTtoUnderline(String src) {
return isBlank(src) ? src : src.replaceAll("[\n|\r|\t]", "_");
}
/**
* <p>Removes control characters (char &lt;= 32) from both
* ends of this String, handling {@code null} by returning
* {@code null}.</p>
*
* @param str the String to be trimmed, may be null
* @return the trimmed string, {@code null} if null String input
*/
public static String trim(final String str) {
return str == null ? null : str.trim();
}
/**
* <p>Returns either the passed in CharSequence, or if the CharSequence is
* whitespace, empty ("") or {@code null}, the value of {@code defaultStr}.</p>
*
* @param <T> the specific kind of CharSequence
* @param str the CharSequence to check, may be null
* @param defaultStr the default CharSequence to return
* if the input is whitespace, empty ("") or {@code null}, may be null
* @return the passed in CharSequence, or the default
*/
public static <T extends CharSequence> T defaultIfBlank(final T str, final T defaultStr) {
return isBlank(str) ? defaultStr : str;
}
/**
* <p>Compares two String, returning {@code true} if they represent
* equal string, ignoring case.</p>
*
* @param str1 the first String, may be null
* @param str2 the second String, may be null
* @return {@code true} if the String are equal, case insensitive, or
* both {@code null}
*/
public static boolean equalsIgnoreCase(String str1, String str2) {
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
}
public static String substringBefore(final String str, final String separator) {
if (isEmpty(str) || separator == null) {
return str;
}
if (separator.isEmpty()) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return str;
}
return str.substring(0, pos);
}
public static String substringAfter(final String str, final String separator) {
if (isEmpty(str)) {
return str;
}
if (separator == null) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return EMPTY;
}
return str.substring(pos + separator.length());
}
public static long strDigitToLong(String str, long defaultValue) {
if (str == null) {
return defaultValue;
} else {
try {
return Long.parseLong(str);
} catch (NumberFormatException var4) {
return defaultValue;
}
}
}
/**
* <p>Joins the elements of the provided Collection into a single String
* containing the provided Collection of elements.</p>
*
* @param collection the collection, may be null
* @param separator the separator
* @return a single String
*/
public static String join(Collection<?> collection, String separator) {
return collection == null ? null : join(collection.iterator(), separator);
}
/**
* <p>Joins the elements of the provided Iterator into a single String
* containing the provided Iterator of elements.</p>
*
* @param iterator the iterator, may be null
* @param separator the separator
* @return a single String
*/
public static String join(Iterator<?> iterator, String separator) {
if (iterator == null) {
return null;
} else if (!iterator.hasNext()) {
return "";
} else {
Object first = iterator.next();
if (!iterator.hasNext()) {
return first == null ? "" : first.toString();
} else {
StringBuilder buf = new StringBuilder(256);
if (first != null) {
buf.append(first);
}
while (iterator.hasNext()) {
if (separator != null) {
buf.append(separator);
}
Object obj = iterator.next();
if (obj != null) {
buf.append(obj);
}
}
return buf.toString();
}
}
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java

@ -33,7 +33,8 @@ import static org.apache.dolphinscheduler.common.Constants.SUBTRACT_STRING;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.AbstractMap; import java.util.AbstractMap;
import java.util.ArrayList; import java.util.ArrayList;

26
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java

@ -42,20 +42,6 @@ public class DateUtilsTest {
Assert.assertEquals("01 09:23:08", readableDate); Assert.assertEquals("01 09:23:08", readableDate);
} }
@Test
public void testConvertTimeStampsToString() {
TimeZone defaultTimeZone = TimeZone.getDefault();
final TimeZone timeZone = TimeZone.getTimeZone("Asia/Shanghai");
TimeZone.setDefault(timeZone);
long timeMillis = 1625989249021L;
Assert.assertEquals("2021-07-11 15:40:49", DateUtils.formatTimeStamp(timeMillis));
DateTimeFormatter testFormatter = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
Assert.assertEquals("2021/07/11 15:40:49", DateUtils.formatTimeStamp(timeMillis, testFormatter));
TimeZone.setDefault(defaultTimeZone);
}
@Test @Test
public void testWeek() { public void testWeek() {
Date curr = DateUtils.stringToDate("2019-02-01 00:00:00"); Date curr = DateUtils.stringToDate("2019-02-01 00:00:00");
@ -69,18 +55,6 @@ public class DateUtilsTest {
} }
@Test
public void diffHours() {
Date d1 = DateUtils.stringToDate("2019-01-28 00:00:00");
Date d2 = DateUtils.stringToDate("2019-01-28 20:00:00");
Assert.assertEquals(DateUtils.diffHours(d1, d2), 20);
Date d3 = DateUtils.stringToDate("2019-01-28 20:00:00");
Assert.assertEquals(DateUtils.diffHours(d3, d2), 0);
Assert.assertEquals(DateUtils.diffHours(d2, d1), 20);
Date d4 = null;
Assert.assertEquals(DateUtils.diffHours(d2, d4), 0);
}
@Test @Test
public void dateToString() { public void dateToString() {
Date d1 = DateUtils.stringToDate("2019-01-28"); Date d1 = DateUtils.stringToDate("2019-01-28");

39
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java

@ -15,11 +15,11 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.io.StringReader;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito;
import java.io.StringReader;
import java.sql.*;
public class ScriptRunnerTest { public class ScriptRunnerTest {
@Test @Test
@ -33,38 +33,5 @@ public class ScriptRunnerTest {
exception = e; exception = e;
} }
Assert.assertNotNull(exception); Assert.assertNotNull(exception);
//connect is not null
runScript("");
}
private void runScript(String dbName) {
try {
Connection conn = Mockito.mock(Connection.class);
Mockito.when(conn.getAutoCommit()).thenReturn(true);
PreparedStatement st = Mockito.mock(PreparedStatement.class);
Mockito.when(conn.createStatement()).thenReturn(st);
ResultSet rs = Mockito.mock(ResultSet.class);
Mockito.when(st.getResultSet()).thenReturn(rs);
ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class);
Mockito.when(rs.getMetaData()).thenReturn(md);
Mockito.when(md.getColumnCount()).thenReturn(2);
Mockito.when(rs.next()).thenReturn(true, false);
ScriptRunner s = new ScriptRunner(conn, true, true);
if (dbName.isEmpty()) {
s.runScript(new StringReader("select 1;"));
} else {
s.runScript(new StringReader("select 1;"), dbName);
}
Mockito.verify(md).getColumnLabel(1);
} catch(Exception e) {
Assert.assertNotNull(e);
}
}
@Test
public void testRunScriptWithDbName() {
//connect is not null
runScript("db_test");
} }
} }

96
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java

@ -1,96 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
public class StringUtilsTest {
@Test
public void testIsNotEmpty() {
//null string
boolean b = StringUtils.isNotEmpty(null);
Assert.assertFalse(b);
//"" string
b = StringUtils.isNotEmpty("");
Assert.assertFalse(b);
//" " string
b = StringUtils.isNotEmpty(" ");
Assert.assertTrue(b);
//"test" string
b = StringUtils.isNotEmpty("test");
Assert.assertTrue(b);
}
@Test
public void testIsNotBlank() {
//null string
boolean b = StringUtils.isNotBlank(null);
Assert.assertFalse(b);
//"" string
b = StringUtils.isNotBlank("");
Assert.assertFalse(b);
//" " string
b = StringUtils.isNotBlank(" ");
Assert.assertFalse(b);
//" test " string
b = StringUtils.isNotBlank(" test ");
Assert.assertTrue(b);
//"test" string
b = StringUtils.isNotBlank("test");
Assert.assertTrue(b);
}
@Test
public void testTrim() {
String trim = StringUtils.trim(null);
Assert.assertNull(trim);
trim = StringUtils.trim(" test ");
Assert.assertEquals("test", trim);
}
@Test
public void testDefaultIfBlank() {
String defaultStr = StringUtils.defaultIfBlank("", "defaultStr");
Assert.assertEquals("defaultStr", defaultStr);
defaultStr = StringUtils.defaultIfBlank("test", "defaultStr");
Assert.assertEquals("test", defaultStr);
}
@Test
public void testJoin() {
List<String> list = new ArrayList<>();
list.add("1");
list.add("3");
list.add("4");
String join = StringUtils.join(list, ",");
Assert.assertEquals("1,3,4", join);
}
}

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.enums.AlertEvent;
import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertWarnLevel; import org.apache.dolphinscheduler.common.enums.AlertWarnLevel;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
@ -33,6 +32,8 @@ import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertMapper; import org.apache.dolphinscheduler.dao.mapper.AlertMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;

9
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java

@ -19,15 +19,18 @@ package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.KeyDeserializer; import com.fasterxml.jackson.databind.KeyDeserializer;
import java.io.IOException;
import java.util.Date;
/** /**
* udf function * udf function

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@ -23,6 +23,9 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.AbstractBaseDao; import org.apache.dolphinscheduler.dao.AbstractBaseDao;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -144,7 +147,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
// Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler // Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true); ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); Reader initSqlReader = new FileReader(mysqlSQLFilePath);
initScriptRunner.runScript(initSqlReader); initScriptRunner.runScript(initSqlReader);
conn.commit(); conn.commit();
@ -189,7 +192,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
conn = dataSource.getConnection(); conn = dataSource.getConnection();
// Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler // Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true); ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath)); Reader initSqlReader = new FileReader(mysqlSQLFilePath);
initScriptRunner.runScript(initSqlReader); initScriptRunner.runScript(initSqlReader);
} catch (IOException e) { } catch (IOException e) {

5
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.Queue;
import java.util.Date; import java.util.Date;
@ -133,7 +132,7 @@ public class QueueMapperTest {
Assert.assertNull(queueMapper.existQueue("queue", null)); Assert.assertNull(queueMapper.existQueue("queue", null));
Assert.assertNull(queueMapper.existQueue(null, "queue")); Assert.assertNull(queueMapper.existQueue(null, "queue"));
Queue queue = insertOne(); Queue queue = insertOne();
Assert.assertTrue(BooleanUtils.isTrue(queueMapper.existQueue(queue.getQueue(), null))); Assert.assertTrue(queueMapper.existQueue(queue.getQueue(), null) == Boolean.TRUE);
Assert.assertTrue(BooleanUtils.isTrue(queueMapper.existQueue(null, queue.getQueueName()))); Assert.assertTrue(queueMapper.existQueue(null, queue.getQueueName()) == Boolean.TRUE);
} }
} }

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java

@ -22,7 +22,8 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils; import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -57,7 +58,7 @@ public class SensitiveDataConverter extends MessageConverter {
String tempLogMsg = oriLogMsg; String tempLogMsg = oriLogMsg;
if (StringUtils.isNotEmpty(tempLogMsg)) { if (!StringUtils.isEmpty(tempLogMsg)) {
tempLogMsg = passwordHandler(pwdPattern, tempLogMsg); tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
} }
return tempLogMsg; return tempLogMsg;

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java

@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EnumUtils; import org.apache.dolphinscheduler.common.utils.EnumUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Resource;
@ -57,6 +56,8 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.TaskPriority; import org.apache.dolphinscheduler.service.queue.TaskPriority;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -334,7 +335,7 @@ public class TaskPriorityQueueConsumer extends Thread {
// whether udf type // whether udf type
boolean udfTypeFlag = EnumUtils.isValidEnum(UdfType.class, sqlParameters.getType()) boolean udfTypeFlag = EnumUtils.isValidEnum(UdfType.class, sqlParameters.getType())
&& StringUtils.isNotEmpty(sqlParameters.getUdfs()); && !StringUtils.isEmpty(sqlParameters.getUdfs());
if (udfTypeFlag) { if (udfTypeFlag) {
String[] udfFunIds = sqlParameters.getUdfs().split(","); String[] udfFunIds = sqlParameters.getUdfs().split(",");

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java

@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.server.master.dispatch; package org.apache.dolphinscheduler.server.master.dispatch;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
@ -27,6 +26,8 @@ import org.apache.dolphinscheduler.server.master.dispatch.executor.ExecutorManag
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager; import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager; import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager;
import org.apache.commons.lang.StringUtils;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.InitializingBean;

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java

@ -20,13 +20,14 @@ package org.apache.dolphinscheduler.server.master.dispatch.host;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.ResInfo; import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
@ -88,7 +89,7 @@ public abstract class CommonHostManager implements HostManager {
protected int getWorkerHostWeightFromHeartbeat(String heartbeat) { protected int getWorkerHostWeightFromHeartbeat(String heartbeat) {
int hostWeight = Constants.DEFAULT_WORKER_HOST_WEIGHT; int hostWeight = Constants.DEFAULT_WORKER_HOST_WEIGHT;
if (StringUtils.isNotEmpty(heartbeat)) { if (!StringUtils.isEmpty(heartbeat)) {
String[] parts = heartbeat.split(Constants.COMMA); String[] parts = heartbeat.split(Constants.COMMA);
if (ResInfo.isNewHeartbeatWithWeight(parts)) { if (ResInfo.isNewHeartbeatWithWeight(parts)) {
hostWeight = Integer.parseInt(parts[10]); hostWeight = Integer.parseInt(parts[10]);

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java

@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
@ -46,6 +45,8 @@ import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.dolphinscheduler.spi.register.RegistryConnectListener; import org.apache.dolphinscheduler.spi.register.RegistryConnectListener;
import org.apache.dolphinscheduler.spi.register.RegistryConnectState; import org.apache.dolphinscheduler.spi.register.RegistryConnectState;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ -160,7 +161,7 @@ public class MasterRegistryClient {
registryClient.getLock(failoverPath); registryClient.getLock(failoverPath);
String serverHost = null; String serverHost = null;
if (StringUtils.isNotEmpty(path)) { if (!StringUtils.isEmpty(path)) {
serverHost = registryClient.getHostByEventDataPath(path); serverHost = registryClient.getHostByEventDataPath(path);
if (StringUtils.isEmpty(serverHost)) { if (StringUtils.isEmpty(serverHost)) {
logger.error("server down error: unknown path: {}", path); logger.error("server down error: unknown path: {}", path);

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup; import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.spi.register.DataChangeEvent;
import org.apache.dolphinscheduler.spi.register.SubscribeListener; import org.apache.dolphinscheduler.spi.register.SubscribeListener;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java

@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.common.enums.StateEventType;
import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand; import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
@ -32,6 +31,8 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;

69
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java

@ -47,9 +47,7 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@ -70,6 +68,8 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@ -182,13 +182,6 @@ public class WorkflowExecuteThread implements Runnable {
*/ */
private NettyExecutorManager nettyExecutorManager; private NettyExecutorManager nettyExecutorManager;
/**
* submit post node
*
* @param parentNodeName parent node name
*/
private Map<String, Object> propToValue = new ConcurrentHashMap<>();
private ConcurrentLinkedQueue<StateEvent> stateEvents = new ConcurrentLinkedQueue<>(); private ConcurrentLinkedQueue<StateEvent> stateEvents = new ConcurrentLinkedQueue<>();
private List<Date> complementListDate = Lists.newLinkedList(); private List<Date> complementListDate = Lists.newLinkedList();
@ -474,21 +467,6 @@ public class WorkflowExecuteThread implements Runnable {
} }
} }
/**
* prepare process parameter
*
* @throws Exception exception
*/
private void prepareProcess() throws Exception {
// gen process dag
buildFlowDag();
// init task queue
initTaskQueue();
logger.info("prepare process :{} end", processInstance.getId());
}
/** /**
* process end handle * process end handle
*/ */
@ -1053,23 +1031,6 @@ public class WorkflowExecuteThread implements Runnable {
return state; return state;
} }
/**
* whether standby task list have retry tasks
*/
private boolean retryTaskExists() {
boolean result = false;
for (Iterator<TaskInstance> iter = readyToSubmitTaskQueue.iterator(); iter.hasNext(); ) {
TaskInstance task = iter.next();
if (task.getState().typeIsFailure()) {
result = true;
break;
}
}
return result;
}
/** /**
* whether complement end * whether complement end
* *
@ -1171,32 +1132,6 @@ public class WorkflowExecuteThread implements Runnable {
return false; return false;
} }
/**
* whether check process time out
*
* @param processInstance task instance
* @return true if time out of process instance > running time of process instance
*/
private boolean checkProcessTimeOut(ProcessInstance processInstance) {
if (processInstance.getTimeout() == 0) {
return false;
}
Date now = new Date();
long runningTime = DateUtils.diffMin(now, processInstance.getStartTime());
return runningTime > processInstance.getTimeout();
}
/**
* whether can submit task to queue
*
* @return boolean
*/
private boolean canSubmitTaskToQueue() {
return OSUtils.checkResource(masterConfig.getMasterMaxCpuloadAvg(), masterConfig.getMasterReservedMemory());
}
/** /**
* close the on going tasks * close the on going tasks
*/ */

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java

@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master.runner.task;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand;
@ -36,6 +35,8 @@ import org.apache.dolphinscheduler.service.queue.TaskPriority;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import org.slf4j.Logger; import org.slf4j.Logger;

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java

@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters;
import org.apache.dolphinscheduler.common.task.switchtask.SwitchResultVo; import org.apache.dolphinscheduler.common.task.switchtask.SwitchResultVo;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@ -35,6 +34,8 @@ import org.apache.dolphinscheduler.server.utils.SwitchTaskUtils;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java

@ -17,7 +17,9 @@
package org.apache.dolphinscheduler.server.monitor; package org.apache.dolphinscheduler.server.monitor;
import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java

@ -17,7 +17,9 @@
package org.apache.dolphinscheduler.server.monitor; package org.apache.dolphinscheduler.server.monitor;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;

300
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java

@ -26,11 +26,12 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.io.File; import java.io.File;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
@ -46,11 +47,7 @@ import org.slf4j.LoggerFactory;
* mainly used to get the start command line of a process. * mainly used to get the start command line of a process.
*/ */
public class ProcessUtils { public class ProcessUtils {
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
/**
* logger
*/
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
/** /**
* Initialization regularization, solve the problem of pre-compilation performance, * Initialization regularization, solve the problem of pre-compilation performance,
@ -63,233 +60,6 @@ public class ProcessUtils {
*/ */
private static final Pattern WINDOWSATTERN = Pattern.compile("\\w+\\((\\d+)\\)"); private static final Pattern WINDOWSATTERN = Pattern.compile("\\w+\\((\\d+)\\)");
private static final String LOCAL_PROCESS_EXEC = "jdk.lang.Process.allowAmbiguousCommands";
/**
* build command line characters.
*
* @param commandList command list
* @return command
*/
public static String buildCommandStr(List<String> commandList) {
String cmdstr;
String[] cmd = commandList.toArray(new String[0]);
SecurityManager security = System.getSecurityManager();
boolean allowAmbiguousCommands = isAllowAmbiguousCommands(security);
if (allowAmbiguousCommands) {
String executablePath = new File(cmd[0]).getPath();
if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
executablePath = quoteString(executablePath);
}
cmdstr = createCommandLine(
VERIFICATION_LEGACY, executablePath, cmd);
} else {
String executablePath;
try {
executablePath = getExecutablePath(cmd[0]);
} catch (IllegalArgumentException e) {
StringBuilder join = new StringBuilder();
for (String s : cmd) {
join.append(s).append(' ');
}
cmd = getTokensFromCommand(join.toString());
executablePath = getExecutablePath(cmd[0]);
// Check new executable name once more
if (security != null) {
security.checkExec(executablePath);
}
}
cmdstr = createCommandLine(
isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
}
return cmdstr;
}
/**
* check is allow ambiguous commands
*
* @param security security manager
* @return allow ambiguous command flag
*/
private static boolean isAllowAmbiguousCommands(SecurityManager security) {
boolean allowAmbiguousCommands = false;
if (security == null) {
allowAmbiguousCommands = true;
String value = System.getProperty(LOCAL_PROCESS_EXEC);
if (value != null) {
allowAmbiguousCommands = !Constants.STRING_FALSE.equalsIgnoreCase(value);
}
}
return allowAmbiguousCommands;
}
/**
* get executable path.
*
* @param path path
* @return executable path
*/
private static String getExecutablePath(String path) {
boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
return fileToRun.getPath();
}
/**
* whether is shell file.
*
* @param executablePath executable path
* @return true if endsWith .CMD or .BAT
*/
private static boolean isShellFile(String executablePath) {
String upPath = executablePath.toUpperCase();
return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
}
/**
* quote string.
*
* @param arg argument
* @return format arg
*/
private static String quoteString(String arg) {
return '"' + arg + '"';
}
/**
* get tokens from command.
*
* @param command command
* @return token string array
*/
private static String[] getTokensFromCommand(String command) {
ArrayList<String> matchList = new ArrayList<>(8);
Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
while (regexMatcher.find()) {
matchList.add(regexMatcher.group());
}
return matchList.toArray(new String[0]);
}
/**
* Lazy Pattern.
*/
private static class LazyPattern {
/**
* Escape-support version:
* "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
*/
private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
}
/**
* verification cmd bat.
*/
private static final int VERIFICATION_CMD_BAT = 0;
/**
* verification win32.
*/
private static final int VERIFICATION_WIN32 = 1;
/**
* verification legacy.
*/
private static final int VERIFICATION_LEGACY = 2;
/**
* escape verification.
*/
private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
{' ', '\t', '<', '>'}, {' ', '\t'}};
/**
* create command line.
*
* @param verificationType verification type
* @param executablePath executable path
* @param cmd cmd
* @return command line
*/
private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
StringBuilder cmdbuf = new StringBuilder(80);
cmdbuf.append(executablePath);
for (int i = 1; i < cmd.length; ++i) {
cmdbuf.append(' ');
String s = cmd[i];
if (needsEscaping(verificationType, s)) {
cmdbuf.append('"').append(s);
if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
cmdbuf.append('\\');
}
cmdbuf.append('"');
} else {
cmdbuf.append(s);
}
}
return cmdbuf.toString();
}
/**
* whether is quoted.
*
* @param noQuotesInside no quotes inside
* @param arg arg
* @param errorMessage error message
* @return boolean
*/
private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
int lastPos = arg.length() - 1;
if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
// The argument has already been quoted.
if (noQuotesInside && arg.indexOf('"', 1) != lastPos) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
return true;
}
if (noQuotesInside && arg.indexOf('"') >= 0) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
return false;
}
/**
* whether needs escaping.
*
* @param verificationType verification type
* @param arg arg
* @return boolean
*/
private static boolean needsEscaping(int verificationType, String arg) {
boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
if (!argIsQuoted) {
char[] testEscape = ESCAPE_VERIFICATION[verificationType];
for (char c : testEscape) {
if (arg.indexOf(c) >= 0) {
return true;
}
}
}
return false;
}
/** /**
* kill yarn application. * kill yarn application.
* *
@ -299,21 +69,21 @@ public class ProcessUtils {
* @param executePath execute path * @param executePath execute path
*/ */
public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) { public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) {
if (CollectionUtils.isNotEmpty(appIds)) { if (appIds == null || appIds.isEmpty()) {
return;
}
for (String appId : appIds) { for (String appId : appIds) {
try { try {
ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId); ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
if (!applicationStatus.typeIsFinished()) { if (!applicationStatus.typeIsFinished()) {
String commandFile = String String commandFile = String.format("%s/%s.kill", executePath, appId);
.format("%s/%s.kill", executePath, appId); String cmd = getKerberosInitCommand() + "yarn application -kill " + appId;
String cmd = getKerberosInitCommand() + "yarn application -kill " + appId; execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd);
execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd);
}
} catch (Exception e) {
logger.error(String.format("Get yarn application app id [%s] status failed: [%s]", appId, e.getMessage()));
} }
} catch (Exception e) {
logger.error("Get yarn application app id [{}}] status failed", appId, e);
} }
} }
} }
@ -321,15 +91,15 @@ public class ProcessUtils {
/** /**
* get kerberos init command * get kerberos init command
*/ */
public static String getKerberosInitCommand() { static String getKerberosInitCommand() {
logger.info("get kerberos init command"); logger.info("get kerberos init command");
StringBuilder kerberosCommandBuilder = new StringBuilder(); StringBuilder kerberosCommandBuilder = new StringBuilder();
boolean hadoopKerberosState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false); boolean hadoopKerberosState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
if (hadoopKerberosState) { if (hadoopKerberosState) {
kerberosCommandBuilder.append("export KRB5_CONFIG=") kerberosCommandBuilder.append("export KRB5_CONFIG=")
.append(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)) .append(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))
.append("\n\n") .append("\n\n")
.append(String.format("kinit -k -t %s %s || true",PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME))) .append(String.format("kinit -k -t %s %s || true", PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)))
.append("\n\n"); .append("\n\n");
logger.info("kerberos init command: {}", kerberosCommandBuilder); logger.info("kerberos init command: {}", kerberosCommandBuilder);
} }
@ -360,7 +130,7 @@ public class ProcessUtils {
File f = new File(commandFile); File f = new File(commandFile);
if (!f.exists()) { if (!f.exists()) {
FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8); org.apache.commons.io.FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
} }
String runCmd = String.format("%s %s", Constants.SH, commandFile); String runCmd = String.format("%s %s", Constants.SH, commandFile);
@ -372,35 +142,6 @@ public class ProcessUtils {
} }
} }
/**
* kill tasks according to different task types.
*
* @param taskExecutionContext taskExecutionContext
*/
public static void kill(TaskExecutionContext taskExecutionContext) {
try {
int processId = taskExecutionContext.getProcessId();
if (processId == 0) {
logger.error("process kill failed, process id :{}, task id:{}",
processId, taskExecutionContext.getTaskInstanceId());
return;
}
String pidsStr = getPidsStr(processId);
if (StringUtils.isNotEmpty(pidsStr)) {
String cmd = String.format("kill -9 %s", pidsStr);
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", processId, cmd);
OSUtils.exeCmd(cmd);
}
} catch (Exception e) {
logger.error("kill task failed", e);
}
// find log and kill yarn job
killYarnJob(taskExecutionContext);
}
/** /**
* get pids str. * get pids str.
* *
@ -436,6 +177,7 @@ public class ProcessUtils {
/** /**
* find logs and kill yarn tasks. * find logs and kill yarn tasks.
*
* @param taskExecutionContext taskExecutionContext * @param taskExecutionContext taskExecutionContext
* @return yarn application ids * @return yarn application ids
*/ */
@ -448,7 +190,7 @@ public class ProcessUtils {
Constants.RPC_PORT, Constants.RPC_PORT,
taskExecutionContext.getLogPath()); taskExecutionContext.getLogPath());
} }
if (StringUtils.isNotEmpty(log)) { if (!StringUtils.isEmpty(log)) {
if (StringUtils.isEmpty(taskExecutionContext.getExecutePath())) { if (StringUtils.isEmpty(taskExecutionContext.getExecutePath())) {
taskExecutionContext.setExecutePath(FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(), taskExecutionContext.setExecutePath(FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(),
taskExecutionContext.getProcessDefineCode(), taskExecutionContext.getProcessDefineCode(),

7
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java

@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand;
@ -40,6 +39,8 @@ import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
import org.apache.commons.lang.StringUtils;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -122,7 +123,7 @@ public class TaskKillProcessor implements NettyRequestProcessor {
} }
String pidsStr = ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()); String pidsStr = ProcessUtils.getPidsStr(taskExecutionContext.getProcessId());
if (StringUtils.isNotEmpty(pidsStr)) { if (!StringUtils.isEmpty(pidsStr)) {
String cmd = String.format("kill -9 %s", pidsStr); String cmd = String.format("kill -9 %s", pidsStr);
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd); logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd);
@ -180,7 +181,7 @@ public class TaskKillProcessor implements NettyRequestProcessor {
logger.info("view log host : {},logPath : {}", host, logPath); logger.info("view log host : {},logPath : {}", host, logPath);
String log = logClient.viewLog(host, Constants.RPC_PORT, logPath); String log = logClient.viewLog(host, Constants.RPC_PORT, logPath);
List<String> appIds = Collections.emptyList(); List<String> appIds = Collections.emptyList();
if (StringUtils.isNotEmpty(log)) { if (!StringUtils.isEmpty(log)) {
appIds = LoggerUtils.getAppIds(log, logger); appIds = LoggerUtils.getAppIds(log, logger);
if (StringUtils.isEmpty(executePath)) { if (StringUtils.isEmpty(executePath)) {
logger.error("task instance execute path is empty"); logger.error("task instance execute path is empty");

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java

@ -26,12 +26,13 @@ import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
import org.apache.dolphinscheduler.server.registry.HeartBeatTask; import org.apache.dolphinscheduler.server.registry.HeartBeatTask;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.service.registry.RegistryClient; import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.lang.StringUtils;
import java.util.Date; import java.util.Date;
import java.util.Set; import java.util.Set;
import java.util.StringJoiner; import java.util.StringJoiner;

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.RetryerUtils; import org.apache.dolphinscheduler.common.utils.RetryerUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand;
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
@ -42,6 +41,7 @@ import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;

14
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java

@ -17,14 +17,16 @@
package org.apache.dolphinscheduler.server.log; package org.apache.dolphinscheduler.server.log;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -46,7 +48,7 @@ public class LoggerServerTest {
@Test @Test
public void testRollViewLog() throws IOException { public void testRollViewLog() throws IOException {
String expectedTmpDemoString = "testRolloViewLog"; String expectedTmpDemoString = "testRolloViewLog";
FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset()); org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset());
String resultTmpDemoString = this.logClientService.rollViewLog( String resultTmpDemoString = this.logClientService.rollViewLog(
"localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000); "localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000);
@ -59,7 +61,7 @@ public class LoggerServerTest {
@Test @Test
public void testRemoveTaskLog() throws IOException { public void testRemoveTaskLog() throws IOException {
String expectedTmpRemoveString = "testRemoveTaskLog"; String expectedTmpRemoveString = "testRemoveTaskLog";
FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset()); org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset());
Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt"); Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt");

5
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java

@ -17,12 +17,13 @@
package org.apache.dolphinscheduler.server.master.dispatch.host; package org.apache.dolphinscheduler.server.master.dispatch.host;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils; import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -59,7 +60,7 @@ public class RoundRobinHostManagerTest {
Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22")); Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22"));
ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000);
Host host = roundRobinHostManager.select(context); Host host = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress())); Assert.assertTrue(!StringUtils.isEmpty(host.getAddress()));
Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22")); Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22"));
} }
} }

48
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -64,53 +63,6 @@ public class ProcessUtilsTest {
Assert.assertEquals("", pidListMac); Assert.assertEquals("", pidListMac);
} }
@Test
public void testBuildCommandStr() {
List<String> commands = new ArrayList<>();
commands.add("sudo");
commands.add("-u");
commands.add("tenantCode");
//allowAmbiguousCommands false
Assert.assertEquals("sudo -u tenantCode", ProcessUtils.buildCommandStr(commands));
//quota
commands.clear();
commands.add("\"sudo\"");
Assert.assertEquals("\"sudo\"", ProcessUtils.buildCommandStr(commands));
//allowAmbiguousCommands true
commands.clear();
commands.add("sudo");
System.setProperty("jdk.lang.Process.allowAmbiguousCommands", "false");
Assert.assertEquals("\"sudo\"", ProcessUtils.buildCommandStr(commands));
}
@Test
public void testKill() {
//get taskExecutionContext
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
//process id eq 0
taskExecutionContext.setProcessId(0);
ProcessUtils.kill(taskExecutionContext);
//process id not eq 0
taskExecutionContext.setProcessId(1);
PowerMockito.mockStatic(OSUtils.class);
try {
when(OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, 1))).thenReturn("1111");
when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, 1))).thenReturn("1111");
when(OSUtils.exeCmd("sudo -u tenantCode kill -9")).thenReturn("1111");
} catch (Exception e) {
e.printStackTrace();
}
taskExecutionContext.setHost("127.0.0.1:8888");
taskExecutionContext.setLogPath("/log/1.log");
taskExecutionContext.setTenantCode("tenantCode");
ProcessUtils.kill(taskExecutionContext);
Assert.assertEquals(1, taskExecutionContext.getProcessId());
}
@Test @Test
public void testGetKerberosInitCommand() { public void testGetKerberosInitCommand() {
PowerMockito.mockStatic(PropertyUtils.class); PowerMockito.mockStatic(PropertyUtils.class);

5
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -59,7 +59,6 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException; import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.DagData; import org.apache.dolphinscheduler.dao.entity.DagData;
@ -106,6 +105,8 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
@ -127,8 +128,6 @@ import org.springframework.transaction.annotation.Transactional;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.facebook.presto.jdbc.internal.guava.collect.Lists; import com.facebook.presto.jdbc.internal.guava.collect.Lists;
import com.cronutils.model.Cron;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
/** /**

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java

@ -63,13 +63,13 @@ import static org.quartz.TriggerBuilder.newTrigger;
import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.service.exceptions.ServiceException; import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;

3
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java

@ -23,7 +23,6 @@ import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHED
import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader; import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader;
import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig; import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig;
import org.apache.dolphinscheduler.spi.register.Registry; import org.apache.dolphinscheduler.spi.register.Registry;
@ -32,6 +31,8 @@ import org.apache.dolphinscheduler.spi.register.RegistryException;
import org.apache.dolphinscheduler.spi.register.RegistryPluginManager; import org.apache.dolphinscheduler.spi.register.RegistryPluginManager;
import org.apache.dolphinscheduler.spi.register.SubscribeListener; import org.apache.dolphinscheduler.spi.register.SubscribeListener;
import org.apache.commons.lang.StringUtils;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;

3
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java

@ -32,7 +32,8 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.ResInfo; import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;

56
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

@ -23,7 +23,6 @@ import static org.apache.dolphinscheduler.spi.task.TaskConstants.SH;
import org.apache.dolphinscheduler.plugin.task.util.LoggerUtils; import org.apache.dolphinscheduler.plugin.task.util.LoggerUtils;
import org.apache.dolphinscheduler.plugin.task.util.OSUtils; import org.apache.dolphinscheduler.plugin.task.util.OSUtils;
import org.apache.dolphinscheduler.plugin.task.util.ThreadUtils;
import org.apache.dolphinscheduler.spi.task.TaskConstants; import org.apache.dolphinscheduler.spi.task.TaskConstants;
import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest; import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
@ -41,6 +40,8 @@ import java.util.Collections;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@ -48,6 +49,8 @@ import java.util.regex.Pattern;
import org.slf4j.Logger; import org.slf4j.Logger;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/** /**
* abstract command executor * abstract command executor
*/ */
@ -132,7 +135,6 @@ public abstract class AbstractCommandExecutor {
processBuilder.command(command); processBuilder.command(command);
process = processBuilder.start(); process = processBuilder.start();
// print command
printCommand(command); printCommand(command);
} }
@ -159,7 +161,7 @@ public abstract class AbstractCommandExecutor {
// parse process output // parse process output
parseProcessOutput(process); parseProcessOutput(process);
Integer processId = getProcessId(process); int processId = getProcessId(process);
result.setProcessId(processId); result.setProcessId(processId);
@ -242,7 +244,6 @@ public abstract class AbstractCommandExecutor {
* *
* @param processId process id * @param processId process id
* @return process is alive * @return process is alive
* @throws InterruptedException interrupted exception
*/ */
private boolean softKill(int processId) { private boolean softKill(int processId) {
@ -281,20 +282,8 @@ public abstract class AbstractCommandExecutor {
} }
} }
/**
* print command
*
* @param commands process builder
*/
private void printCommand(List<String> commands) { private void printCommand(List<String> commands) {
String cmdStr; logger.info("task run command: {}", String.join(" ", commands));
try {
cmdStr = ProcessUtils.buildCommandStr(commands);
logger.info("task run command:\n{}", cmdStr);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
} }
/** /**
@ -320,7 +309,7 @@ public abstract class AbstractCommandExecutor {
*/ */
private void parseProcessOutput(Process process) { private void parseProcessOutput(Process process) {
String threadLoggerInfoName = String.format(LoggerUtils.TASK_LOGGER_THREAD_NAME + "-%s", taskRequest.getTaskAppId()); String threadLoggerInfoName = String.format(LoggerUtils.TASK_LOGGER_THREAD_NAME + "-%s", taskRequest.getTaskAppId());
ExecutorService getOutputLogService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName + "-" + "getOutputLogService"); ExecutorService getOutputLogService = newDaemonSingleThreadExecutor(threadLoggerInfoName + "-" + "getOutputLogService");
getOutputLogService.submit(() -> { getOutputLogService.submit(() -> {
try (BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { try (BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
String line; String line;
@ -343,7 +332,7 @@ public abstract class AbstractCommandExecutor {
getOutputLogService.shutdown(); getOutputLogService.shutdown();
ExecutorService parseProcessOutputExecutorService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName); ExecutorService parseProcessOutputExecutorService = newDaemonSingleThreadExecutor(threadLoggerInfoName);
parseProcessOutputExecutorService.submit(() -> { parseProcessOutputExecutorService.submit(() -> {
try { try {
long lastFlushTime = System.currentTimeMillis(); long lastFlushTime = System.currentTimeMillis();
@ -364,10 +353,6 @@ public abstract class AbstractCommandExecutor {
parseProcessOutputExecutorService.shutdown(); parseProcessOutputExecutorService.shutdown();
} }
public int getProcessId() {
return getProcessId(process);
}
/** /**
* get app links * get app links
* *
@ -378,7 +363,7 @@ public abstract class AbstractCommandExecutor {
List<String> logs = convertFile2List(logPath); List<String> logs = convertFile2List(logPath);
List<String> appIds = new ArrayList<>(); List<String> appIds = new ArrayList<>();
/** /*
* analysis log?get submited yarn application id * analysis log?get submited yarn application id
*/ */
for (String log : logs) { for (String log : logs) {
@ -398,7 +383,7 @@ public abstract class AbstractCommandExecutor {
* @return line list * @return line list
*/ */
private List<String> convertFile2List(String filename) { private List<String> convertFile2List(String filename) {
List lineList = new ArrayList<String>(100); List<String> lineList = new ArrayList<>(100);
File file = new File(filename); File file = new File(filename);
if (!file.exists()) { if (!file.exists()) {
@ -477,12 +462,11 @@ public abstract class AbstractCommandExecutor {
private long flush(long lastFlushTime) { private long flush(long lastFlushTime) {
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
/** /*
* when log buffer siz or flush time reach condition , then flush * when log buffer siz or flush time reach condition , then flush
*/ */
if (logBuffer.size() >= TaskConstants.DEFAULT_LOG_ROWS_NUM || now - lastFlushTime > TaskConstants.DEFAULT_LOG_FLUSH_INTERVAL) { if (logBuffer.size() >= TaskConstants.DEFAULT_LOG_ROWS_NUM || now - lastFlushTime > TaskConstants.DEFAULT_LOG_FLUSH_INTERVAL) {
lastFlushTime = now; lastFlushTime = now;
/** log handle */
logHandler.accept(logBuffer); logHandler.accept(logBuffer);
logBuffer.clear(); logBuffer.clear();
@ -490,21 +474,15 @@ public abstract class AbstractCommandExecutor {
return lastFlushTime; return lastFlushTime;
} }
protected List<String> commandOptions() {
return Collections.emptyList();
}
protected abstract String buildCommandFilePath(); protected abstract String buildCommandFilePath();
protected abstract String commandInterpreter();
protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException; protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException;
public String getTaskResultString() { ExecutorService newDaemonSingleThreadExecutor(String threadName) {
return taskResultString; ThreadFactory threadFactory = new ThreadFactoryBuilder()
} .setDaemon(true)
.setNameFormat(threadName)
public void setTaskResultString(String taskResultString) { .build();
this.taskResultString = taskResultString; return Executors.newSingleThreadExecutor(threadFactory);
} }
} }

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save