Browse Source

Clean up third-party files that are not mentioned in LICENSE (#6165)

2.0.7-release
kezhenxu94 3 years ago committed by GitHub
parent
commit
ea2a9dbcb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 37
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java
  2. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java
  3. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java
  4. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java
  5. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java
  6. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
  7. 89
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
  8. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java
  9. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java
  10. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java
  11. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
  12. 7
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java
  13. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
  14. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
  15. 14
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
  16. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  17. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
  18. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  19. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
  20. 10
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java
  21. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
  22. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java
  23. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
  24. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
  25. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
  26. 11
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
  27. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  28. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
  29. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
  30. 23
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java
  31. 34
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java
  32. 226
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java
  33. 5
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
  34. 3
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  35. 7
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java
  36. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java
  37. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  38. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java
  39. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java
  40. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java
  41. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java
  42. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java
  43. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java
  44. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java
  45. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java
  46. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java
  47. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
  48. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java
  49. 9
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java
  50. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java
  51. 118
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java
  52. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
  53. 25
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java
  54. 33
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java
  55. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
  56. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  57. 99
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  58. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
  59. 185
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
  60. 1
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  61. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  62. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  63. 6
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  64. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java
  65. 2
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
  66. 316
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
  67. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
  68. 226
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  69. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
  70. 26
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java
  71. 39
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java
  72. 96
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java
  73. 3
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java
  74. 9
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java
  75. 7
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
  76. 7
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java
  77. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
  78. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
  79. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java
  80. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
  81. 5
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
  82. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
  83. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
  84. 69
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
  85. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
  86. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java
  87. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
  88. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java
  89. 300
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
  90. 7
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
  91. 3
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
  92. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
  93. 14
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
  94. 5
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
  95. 48
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
  96. 5
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
  97. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java
  98. 3
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java
  99. 3
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
  100. 58
      dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
  101. Some files were not shown because too many files have changed in this diff Show More

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java

@ -16,25 +16,44 @@
*/
package org.apache.dolphinscheduler.api.configuration;
import com.google.common.collect.Multimap;
import io.swagger.models.*;
import io.swagger.models.parameters.Parameter;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import static com.google.common.collect.Maps.newTreeMap;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
import org.springframework.context.MessageSource;
import org.springframework.context.annotation.Primary;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Component;
import com.google.common.collect.Multimap;
import io.swagger.models.Contact;
import io.swagger.models.Info;
import io.swagger.models.Model;
import io.swagger.models.Scheme;
import io.swagger.models.Swagger;
import io.swagger.models.Tag;
import io.swagger.models.parameters.Parameter;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.service.ApiListing;
import springfox.documentation.service.Documentation;
import springfox.documentation.service.ResourceListing;
import springfox.documentation.swagger2.mappers.*;
import java.util.*;
import static com.google.common.collect.Maps.newTreeMap;
import springfox.documentation.swagger2.mappers.LicenseMapper;
import springfox.documentation.swagger2.mappers.ModelMapper;
import springfox.documentation.swagger2.mappers.ParameterMapper;
import springfox.documentation.swagger2.mappers.SecurityMapper;
import springfox.documentation.swagger2.mappers.ServiceModelToSwagger2Mapper;
import springfox.documentation.swagger2.mappers.VendorExtensionsMapper;
/**
* application configuration

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java

@ -25,7 +25,8 @@ import static org.apache.dolphinscheduler.common.Constants.HTTP_X_REAL_IP;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat;
import java.util.HashMap;
@ -253,4 +254,4 @@ public class BaseController {
result.setData(list);
return result;
}
}
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoginController.java

@ -28,10 +28,10 @@ import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang.StringUtils;
import java.util.Map;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java

@ -43,10 +43,11 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -588,7 +589,7 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam("codes") String codes) {
Map<String, Object> result = new HashMap<>();
List<String> deleteFailedCodeList = new ArrayList<>();
if (StringUtils.isNotEmpty(codes)) {
if (!StringUtils.isEmpty(codes)) {
String[] processDefinitionCodeArray = codes.split(",");
for (String strProcessDefinitionCode : processDefinitionCodeArray) {
long code = Long.parseLong(strProcessDefinitionCode);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java

@ -37,10 +37,11 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@ -395,7 +396,7 @@ public class ProcessInstanceController extends BaseController {
// task queue
Map<String, Object> result = new HashMap<>();
List<String> deleteFailedIdList = new ArrayList<>();
if (StringUtils.isNotEmpty(processInstanceIds)) {
if (!StringUtils.isEmpty(processInstanceIds)) {
String[] processInstanceIdArray = processInstanceIds.split(",");
for (String strProcessInstanceId : processInstanceIdArray) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@ -52,9 +52,10 @@ import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.util.Map;
import org.slf4j.Logger;

89
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java

@ -21,11 +21,11 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.lang.StringUtils;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@ -36,56 +36,57 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.servlet.HandlerInterceptor;
/**
* login interceptor, must login first
* login interceptor, must log in first
*/
public class LoginHandlerInterceptor implements HandlerInterceptor {
private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptor.class);
private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptor.class);
@Autowired
private UserMapper userMapper;
@Autowired
private UserMapper userMapper;
@Autowired
private Authenticator authenticator;
@Autowired
private Authenticator authenticator;
/**
* Intercept the execution of a handler. Called after HandlerMapping determined
* @param request current HTTP request
* @param response current HTTP response
* @param handler chosen handler to execute, for type and/or instance evaluation
* @return boolean true or false
*/
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
/**
* Intercept the execution of a handler. Called after HandlerMapping determined
*
* @param request current HTTP request
* @param response current HTTP response
* @param handler chosen handler to execute, for type and/or instance evaluation
* @return boolean true or false
*/
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
// get token
String token = request.getHeader("token");
User user = null;
if (StringUtils.isEmpty(token)){
user = authenticator.getAuthUser(request);
// if user is null
if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("user does not exist");
return false;
}
}else {
user = userMapper.queryUserByToken(token);
if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("user token has expired");
return false;
}
}
// get token
String token = request.getHeader("token");
User user;
if (StringUtils.isEmpty(token)) {
user = authenticator.getAuthUser(request);
// if user is null
if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("user does not exist");
return false;
}
} else {
user = userMapper.queryUserByToken(token);
if (user == null) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info("user token has expired");
return false;
}
}
// check user state
if (user.getState() == Flag.NO.ordinal()) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info(Status.USER_DISABLED.getMsg());
return false;
}
// check user state
if (user.getState() == Flag.NO.ordinal()) {
response.setStatus(HttpStatus.SC_UNAUTHORIZED);
logger.info(Status.USER_DISABLED.getMsg());
return false;
}
request.setAttribute(Constants.SESSION_USER, user);
return true;
}
request.setAttribute(Constants.SESSION_USER, user);
return true;
}
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java

@ -18,9 +18,9 @@
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.util.Map;
import java.util.concurrent.ExecutionException;
@ -74,7 +74,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
// tenant-level rate limit
if (trafficConfiguration.isTrafficTenantControlSwitch()) {
String token = request.getHeader("token");
if (StringUtils.isNotEmpty(token)) {
if (!StringUtils.isEmpty(token)) {
RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token);
if (!tenantRateLimiter.tryAcquire()) {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java

@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.api.security;
import org.apache.dolphinscheduler.api.security.impl.ldap.LdapAuthenticator;
import org.apache.dolphinscheduler.api.security.impl.pwd.PasswordAuthenticator;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java

@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.api.service.AlertGroupService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.vo.AlertGroupVo;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -195,7 +195,7 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
Date now = new Date();
if (StringUtils.isNotEmpty(groupName)) {
if (!StringUtils.isEmpty(groupName)) {
alertGroup.setGroupName(groupName);
}
alertGroup.setDescription(desc);
@ -248,6 +248,6 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup
*/
@Override
public boolean existGroupName(String groupName) {
return BooleanUtils.isTrue(alertGroupMapper.existGroupName(groupName));
return alertGroupMapper.existGroupName(groupName) == Boolean.TRUE;
}
}

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java

@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
@ -85,7 +84,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A
Map<String, Object> result = new HashMap<>();
if (BooleanUtils.isTrue(alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()))) {
if (alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()) == Boolean.TRUE) {
putMsg(result, Status.PLUGIN_INSTANCE_ALREADY_EXIT);
return result;
}
@ -184,7 +183,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A
@Override
public boolean checkExistPluginInstanceName(String pluginInstanceName) {
return BooleanUtils.isTrue(alertPluginInstanceMapper.existInstanceName(pluginInstanceName));
return alertPluginInstanceMapper.existInstanceName(pluginInstanceName) == Boolean.TRUE;
}
@Override

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java

@ -24,9 +24,10 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.Date;
@ -157,7 +158,7 @@ public class BaseServiceImpl implements BaseService {
public Map<String, Object> checkAndParseDateParameters(String startDateStr, String endDateStr) {
Map<String, Object> result = new HashMap<>();
Date start = null;
if (StringUtils.isNotEmpty(startDateStr)) {
if (!StringUtils.isEmpty(startDateStr)) {
start = DateUtils.getScheduleDate(startDateStr);
if (Objects.isNull(start)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.START_END_DATE);
@ -167,7 +168,7 @@ public class BaseServiceImpl implements BaseService {
result.put(Constants.START_TIME, start);
Date end = null;
if (StringUtils.isNotEmpty(endDateStr)) {
if (!StringUtils.isEmpty(endDateStr)) {
end = DateUtils.getScheduleDate(endDateStr);
if (Objects.isNull(end)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.START_END_DATE);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java

@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TriFunction;
import org.apache.dolphinscheduler.dao.entity.CommandCount;
import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser;
@ -43,6 +42,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
@ -144,7 +145,7 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
Date start = null;
Date end = null;
if (StringUtils.isNotEmpty(startDate) && StringUtils.isNotEmpty(endDate)) {
if (!StringUtils.isEmpty(startDate) && !StringUtils.isEmpty(endDate)) {
start = DateUtils.getScheduleDate(startDate);
end = DateUtils.getScheduleDate(endDate);
if (Objects.isNull(start) || Objects.isNull(end)) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@ -28,12 +28,13 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Date;

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@ -37,6 +36,7 @@ import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapp
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.commons.collections4.SetUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Date;
@ -126,11 +126,11 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
}
if (environmentMapper.insert(env) > 0) {
if (StringUtils.isNotEmpty(workerGroups)) {
if (!StringUtils.isEmpty(workerGroups)) {
List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){});
if (CollectionUtils.isNotEmpty(workerGroupList)) {
workerGroupList.stream().forEach(workerGroup -> {
if (StringUtils.isNotEmpty(workerGroup)) {
if (!StringUtils.isEmpty(workerGroup)) {
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(env.getCode());
relation.setWorkerGroup(workerGroup);
@ -338,7 +338,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
}
Set<String> workerGroupSet;
if (StringUtils.isNotEmpty(workerGroups)) {
if (!StringUtils.isEmpty(workerGroups)) {
workerGroupSet = JSONUtils.parseObject(workerGroups, new TypeReference<Set<String>>() {});
} else {
workerGroupSet = new TreeSet<>();
@ -370,14 +370,14 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
int update = environmentMapper.update(env, new UpdateWrapper<Environment>().lambda().eq(Environment::getCode,code));
if (update > 0) {
deleteWorkerGroupSet.stream().forEach(key -> {
if (StringUtils.isNotEmpty(key)) {
if (!StringUtils.isEmpty(key)) {
relationMapper.delete(new QueryWrapper<EnvironmentWorkerGroupRelation>()
.lambda()
.eq(EnvironmentWorkerGroupRelation::getEnvironmentCode,code));
}
});
addWorkerGroupSet.stream().forEach(key -> {
if (StringUtils.isNotEmpty(key)) {
if (!StringUtils.isEmpty(key)) {
EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation();
relation.setEnvironmentCode(code);
relation.setWorkerGroup(key);
@ -448,7 +448,7 @@ public class EnvironmentServiceImpl extends BaseServiceImpl implements Environme
putMsg(result, Status.ENVIRONMENT_CONFIG_IS_NULL);
return result;
}
if (StringUtils.isNotEmpty(workerGroups)) {
if (!StringUtils.isEmpty(workerGroups)) {
List<String> workerGroupList = JSONUtils.parseObject(workerGroups, new TypeReference<List<String>>(){});
if (Objects.isNull(workerGroupList)) {
putMsg(result, Status.ENVIRONMENT_WORKER_GROUPS_IS_INVALID);

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@ -42,7 +42,6 @@ import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@ -59,6 +58,7 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Date;
@ -416,7 +416,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
//To add startParams only when repeat running is needed
Map<String, Object> cmdParam = new HashMap<>();
cmdParam.put(CMD_PARAM_RECOVER_PROCESS_ID_STRING, instanceId);
if (StringUtils.isNotEmpty(startParams)) {
if (!StringUtils.isEmpty(startParams)) {
cmdParam.put(CMD_PARAM_START_PARAMS, startParams);
}
@ -527,7 +527,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
command.setFailureStrategy(failureStrategy);
}
if (StringUtils.isNotEmpty(startNodeList)) {
if (!StringUtils.isEmpty(startNodeList)) {
cmdParam.put(CMD_PARAM_START_NODE_NAMES, startNodeList);
}
if (warningType != null) {
@ -545,7 +545,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
Date start = null;
Date end = null;
if (StringUtils.isNotEmpty(schedule)) {
if (!StringUtils.isEmpty(schedule)) {
String[] interval = schedule.split(",");
if (interval.length == 2) {
start = DateUtils.getScheduleDate(interval[0]);

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java

@ -23,12 +23,13 @@ import org.apache.dolphinscheduler.api.service.LoggerService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.ArrayUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.nio.charset.StandardCharsets;
import java.util.Objects;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -44,7 +44,6 @@ import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
@ -71,6 +70,8 @@ import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.permission.PermissionCheck;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@ -49,7 +49,6 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@ -68,6 +67,8 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -653,7 +654,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion());
String localParams = JSONUtils.getNodeString(taskDefinitionLog.getTaskParams(), LOCAL_PARAMS);
if (StringUtils.isNotEmpty(localParams)) {
if (!StringUtils.isEmpty(localParams)) {
localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams);
List<Property> localParamsList = JSONUtils.toList(localParams, Property.class);

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java

@ -22,13 +22,13 @@ import org.apache.dolphinscheduler.api.service.QueueService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -271,7 +271,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
* @return true if the queue not exists, otherwise return false
*/
private boolean checkQueueExist(String queue) {
return BooleanUtils.isTrue(queueMapper.existQueue(queue, null));
return queueMapper.existQueue(queue, null) == Boolean.TRUE;
}
/**
@ -282,7 +282,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
* @return true if the queue name not exists, otherwise return false
*/
private boolean checkQueueNameExist(String queueName) {
return BooleanUtils.isTrue(queueMapper.existQueue(null, queueName));
return queueMapper.existQueue(null, queueName) == Boolean.TRUE;
}
/**
@ -294,7 +294,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService {
* @return true if need to update user
*/
private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) {
return !oldQueue.equals(newQueue) && BooleanUtils.isTrue(userMapper.existUser(oldQueue));
return !oldQueue.equals(newQueue) && userMapper.existUser(oldQueue) == Boolean.TRUE;
}
}

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@ -34,13 +34,11 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
import org.apache.dolphinscheduler.dao.entity.Tenant;
@ -55,6 +53,7 @@ import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.text.MessageFormat;
@ -253,7 +252,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
*/
private boolean checkResourceExists(String fullName, int userId, int type) {
Boolean existResource = resourcesMapper.existResource(fullName, userId, type);
return BooleanUtils.isTrue(existResource);
return existResource == Boolean.TRUE;
}
/**

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SchedulerServiceImpl.java

@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.Schedule;
@ -48,6 +47,8 @@ import org.apache.dolphinscheduler.service.quartz.ProcessScheduleJob;
import org.apache.dolphinscheduler.service.quartz.QuartzExecutors;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.commons.lang.StringUtils;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
@ -250,7 +251,7 @@ public class SchedulerServiceImpl extends BaseServiceImpl implements SchedulerSe
Date now = new Date();
// updateProcessInstance param
if (StringUtils.isNotEmpty(scheduleExpression)) {
if (!StringUtils.isEmpty(scheduleExpression)) {
ScheduleParam scheduleParam = JSONUtils.parseObject(scheduleExpression, ScheduleParam.class);
if (DateUtils.differSec(scheduleParam.getStartTime(), scheduleParam.getEndTime()) == 0) {
logger.warn("The start time must not be the same as the end");

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java

@ -21,11 +21,12 @@ import org.apache.dolphinscheduler.api.controller.BaseController;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.List;
import java.util.UUID;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@ -40,6 +39,8 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java

@ -23,11 +23,9 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
@ -37,6 +35,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -202,7 +202,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
Date now = new Date();
if (StringUtils.isNotEmpty(tenantCode)) {
if (!StringUtils.isEmpty(tenantCode)) {
tenant.setTenantCode(tenantCode);
}
@ -322,6 +322,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
*/
private boolean checkTenantExists(String tenantCode) {
Boolean existTenant = tenantMapper.existTenant(tenantCode);
return BooleanUtils.isTrue(existTenant);
return existTenant == Boolean.TRUE;
}
}

11
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
@ -32,6 +31,8 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -113,10 +114,10 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setUserId(loginUser.getId());
udf.setFuncName(funcName);
udf.setClassName(className);
if (StringUtils.isNotEmpty(argTypes)) {
if (!StringUtils.isEmpty(argTypes)) {
udf.setArgTypes(argTypes);
}
if (StringUtils.isNotEmpty(database)) {
if (!StringUtils.isEmpty(database)) {
udf.setDatabase(database);
}
udf.setDescription(desc);
@ -221,7 +222,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setFuncName(funcName);
udf.setClassName(className);
udf.setArgTypes(argTypes);
if (StringUtils.isNotEmpty(database)) {
if (!StringUtils.isEmpty(database)) {
udf.setDatabase(database);
}
udf.setDescription(desc);
@ -329,4 +330,4 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
return result;
}
}
}

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -33,7 +33,6 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.DatasourceUser;
import org.apache.dolphinscheduler.dao.entity.Project;
@ -56,6 +55,8 @@ import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java

@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessLineage;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
@ -28,6 +27,8 @@ import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.apache.commons.lang.StringUtils;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java

@ -26,13 +26,14 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -188,7 +189,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
if (CollectionUtils.isNotEmpty(workerGroups)) {
List<WorkerGroup> searchValDataList = new ArrayList<>();
if (StringUtils.isNotEmpty(searchVal)) {
if (!StringUtils.isEmpty(searchVal)) {
for (WorkerGroup workerGroup : workerGroups) {
if (workerGroup.getName().contains(searchVal)) {
searchValDataList.add(workerGroup);

23
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java

@ -22,10 +22,11 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.Map;
@ -73,7 +74,7 @@ public class CheckUtils {
*/
public static Map<String, Object> checkDesc(String desc) {
Map<String, Object> result = new HashMap<>();
if (StringUtils.isNotEmpty(desc) && desc.length() > 200) {
if (!StringUtils.isEmpty(desc) && desc.length() > 200) {
result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR);
result.put(Constants.MSG,
MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "desc length"));
@ -90,7 +91,7 @@ public class CheckUtils {
* @return true if other parameters are valid, otherwise return false
*/
public static boolean checkOtherParams(String otherParams) {
return StringUtils.isNotEmpty(otherParams) && !JSONUtils.checkJsonValid(otherParams);
return !StringUtils.isEmpty(otherParams) && !JSONUtils.checkJsonValid(otherParams);
}
/**
@ -100,7 +101,7 @@ public class CheckUtils {
* @return true if password regex valid, otherwise return false
*/
public static boolean checkPassword(String password) {
return StringUtils.isNotEmpty(password) && password.length() >= 2 && password.length() <= 20;
return !StringUtils.isEmpty(password) && password.length() >= 2 && password.length() <= 20;
}
/**
@ -156,20 +157,6 @@ public class CheckUtils {
return false;
}
/**
* check params
*
* @param userName user name
* @param password password
* @param email email
* @param phone phone
* @return true if user parameters are valid, other return false
*/
public static boolean checkUserParams(String userName, String password, String email, String phone) {
return CheckUtils.checkUserName(userName) && CheckUtils.checkEmail(email) && CheckUtils.checkPassword(password)
&& CheckUtils.checkPhone(phone);
}
/**
* regex check
*

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java

@ -60,40 +60,6 @@ public class RegistryCenterUtils {
return registryClient.getServerList(NodeType.WORKER);
}
private static List<ZookeeperRecord> zookeeperInfoList(String zookeeperServers) {
List<ZookeeperRecord> list = new ArrayList<>(5);
/*
if (StringUtils.isNotBlank(zookeeperServers)) {
String[] zookeeperServersArray = zookeeperServers.split(",");
for (String zookeeperServer : zookeeperServersArray) {
ZooKeeperState state = new ZooKeeperState(zookeeperServer);
boolean ok = state.ruok();
if (ok) {
state.getZookeeperInfo();
}
int connections = state.getConnections();
int watches = state.getWatches();
long sent = state.getSent();
long received = state.getReceived();
String mode = state.getMode();
float minLatency = state.getMinLatency();
float avgLatency = state.getAvgLatency();
float maxLatency = state.getMaxLatency();
int nodeCount = state.getNodeCount();
int status = ok ? 1 : 0;
Date date = new Date();
ZookeeperRecord zookeeperRecord = new ZookeeperRecord(zookeeperServer,connections,watches,sent,received,mode,minLatency,avgLatency,maxLatency,nodeCount,status,date);
list.add(zookeeperRecord);
}
}*/
return list;
}
public static Map<String, String> getServerMaps(NodeType nodeType, boolean hostOnly) {
return registryClient.getServerMaps(nodeType, hostOnly);
}

226
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/ZooKeeperState.java

@ -1,226 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.utils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Scanner;
/**
* zookeeper state monitor
*
*/
public class ZooKeeperState {
private static final Logger logger = LoggerFactory.getLogger(ZooKeeperState.class);
private final String host;
private final int port;
private float minLatency = -1, avgLatency = -1, maxLatency = -1;
private long received = -1;
private long sent = -1;
private int outStanding = -1;
private long zxid = -1;
private String mode = null;
private int nodeCount = -1;
private int watches = -1;
private int connections = -1;
public ZooKeeperState(String connectionString) {
String host = connectionString.substring(0,
connectionString.indexOf(':'));
int port = Integer.parseInt(connectionString.substring(connectionString
.indexOf(':') + 1));
this.host = host;
this.port = port;
}
public void getZookeeperInfo() {
String content = cmd("srvr");
if (StringUtils.isNotBlank(content)) {
try (Scanner scannerForStat = new Scanner(content)) {
while (scannerForStat.hasNext()) {
String line = scannerForStat.nextLine();
if (line.startsWith("Latency min/avg/max:")) {
String[] latencys = getStringValueFromLine(line).split("/");
minLatency = Float.parseFloat(latencys[0]);
avgLatency = Float.parseFloat(latencys[1]);
maxLatency = Float.parseFloat(latencys[2]);
} else if (line.startsWith("Received:")) {
received = Long.parseLong(getStringValueFromLine(line));
} else if (line.startsWith("Sent:")) {
sent = Long.parseLong(getStringValueFromLine(line));
} else if (line.startsWith("Outstanding:")) {
outStanding = Integer.parseInt(getStringValueFromLine(line));
} else if (line.startsWith("Zxid:")) {
zxid = Long.parseLong(getStringValueFromLine(line).substring(2), 16);
} else if (line.startsWith("Mode:")) {
mode = getStringValueFromLine(line);
} else if (line.startsWith("Node count:")) {
nodeCount = Integer.parseInt(getStringValueFromLine(line));
}
}
}
}
String wchsText = cmd("wchs");
if (StringUtils.isNotBlank(wchsText)) {
try (Scanner scannerForWchs = new Scanner(wchsText)) {
while (scannerForWchs.hasNext()) {
String line = scannerForWchs.nextLine();
if (line.startsWith("Total watches:")) {
watches = Integer.parseInt(getStringValueFromLine(line));
}
}
}
}
String consText = cmd("cons");
if (StringUtils.isNotBlank(consText)) {
Scanner scannerForCons = new Scanner(consText);
if (StringUtils.isNotBlank(consText)) {
connections = 0;
}
while (scannerForCons.hasNext()) {
@SuppressWarnings("unused")
String line = scannerForCons.nextLine();
++connections;
}
scannerForCons.close();
}
}
public boolean ruok() {
return "imok\n".equals(cmd("ruok"));
}
private String getStringValueFromLine(String line) {
return line.substring(line.indexOf(":") + 1, line.length()).replaceAll(
" ", "").trim();
}
private class SendThread extends Thread {
private String cmd;
private String ret = "";
public SendThread(String cmd) {
this.cmd = cmd;
}
@Override
public void run() {
try {
ret = FourLetterWordMain.send4LetterWord(host, port, cmd);
} catch (Exception e) {
logger.error(e.getMessage(),e);
return;
}
}
}
private String cmd(String cmd) {
final int waitTimeout = 5;
SendThread sendThread = new SendThread(cmd);
sendThread.setName("FourLetterCmd:" + cmd);
sendThread.start();
try {
sendThread.join(waitTimeout * 1000L);
return sendThread.ret;
} catch (InterruptedException e) {
logger.error("send " + cmd + " to server " + host + ":" + port + " failed!", e);
Thread.currentThread().interrupt();
}
return "";
}
public Logger getLogger() {
return logger;
}
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public float getMinLatency() {
return minLatency;
}
public float getAvgLatency() {
return avgLatency;
}
public float getMaxLatency() {
return maxLatency;
}
public long getReceived() {
return received;
}
public long getSent() {
return sent;
}
public int getOutStanding() {
return outStanding;
}
public long getZxid() {
return zxid;
}
public String getMode() {
return mode;
}
public int getNodeCount() {
return nodeCount;
}
public int getWatches() {
return watches;
}
public int getConnections() {
return connections;
}
@Override
public String toString() {
return "ZooKeeperState [host=" + host + ", port=" + port
+ ", minLatency=" + minLatency + ", avgLatency=" + avgLatency
+ ", maxLatency=" + maxLatency + ", received=" + received
+ ", sent=" + sent + ", outStanding=" + outStanding + ", zxid="
+ zxid + ", mode=" + mode + ", nodeCount=" + nodeCount
+ ", watches=" + watches + ", connections="
+ connections + "]";
}
}

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java

@ -23,10 +23,11 @@ import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.Map;
@ -97,7 +98,7 @@ public class AbstractControllerTest {
String session = sessionService.createSession(loginUser, "127.0.0.1");
sessionId = session;
Assert.assertTrue(StringUtils.isNotEmpty(session));
Assert.assertTrue(!StringUtils.isEmpty(session));
}
public Map<String, Object> success() {

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -30,7 +30,6 @@ import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
@ -47,6 +46,8 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;

7
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java

@ -21,11 +21,12 @@ import org.apache.dolphinscheduler.api.service.impl.SessionServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@ -109,7 +110,7 @@ public class SessionServiceTest {
Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions());
String sessionId = sessionService.createSession(user, ip);
logger.info("createSessionId is " + sessionId);
Assert.assertTrue(StringUtils.isNotEmpty(sessionId));
Assert.assertTrue(!StringUtils.isEmpty(sessionId));
}
/**
@ -143,4 +144,4 @@ public class SessionServiceTest {
sessionList.add(getSession());
return sessionList;
}
}
}

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java

@ -106,7 +106,7 @@ public class FileUtilsTest {
@Test
public void testFile2String() throws IOException {
String content = "123";
org.apache.dolphinscheduler.common.utils.FileUtils.writeStringToFile(new File("/tmp/task.json"),content);
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/task.json"),content);
File file = new File("/tmp/task.json");
FileInputStream fileInputStream = new FileInputStream("/tmp/task.json");
@ -121,4 +121,4 @@ public class FileUtilsTest {
Assert.assertTrue(delete);
}
}
}

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.regex.Pattern;
@ -1095,8 +1096,8 @@ public final class Constants {
/**
* docker & kubernetes
*/
public static final boolean DOCKER_MODE = StringUtils.isNotEmpty(System.getenv("DOCKER"));
public static final boolean KUBERNETES_MODE = StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && StringUtils.isNotEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
public static final boolean DOCKER_MODE = !StringUtils.isEmpty(System.getenv("DOCKER"));
public static final boolean KUBERNETES_MODE = !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
/**
* task parameter keys

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java

@ -24,9 +24,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -83,7 +83,7 @@ public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam;
String jdbcUrl = clickhouseConnectionParam.getJdbcUrl();
if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) {
if (!StringUtils.isEmpty(clickhouseConnectionParam.getOther())) {
jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther());
}
return jdbcUrl;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -84,7 +84,7 @@ public class Db2DatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) {
if (!StringUtils.isEmpty(db2ConnectionParam.getOther())) {
return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther());
}
return db2ConnectionParam.getJdbcUrl();

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java

@ -26,9 +26,9 @@ import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.sql.Connection;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -102,7 +102,7 @@ public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam;
String jdbcUrl = mysqlConnectionParam.getJdbcUrl();
if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) {
if (!StringUtils.isEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS);
}
return String.format("%s?%s", jdbcUrl, APPEND_PARAMS);

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java

@ -26,9 +26,9 @@ import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -99,7 +99,7 @@ public class OracleDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) {
if (!StringUtils.isEmpty(oracleConnectionParam.getOther())) {
return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther());
}
return oracleConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -84,7 +84,7 @@ public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) {
if (!StringUtils.isEmpty(postgreSqlConnectionParam.getOther())) {
return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther());
}
return postgreSqlConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -86,7 +86,7 @@ public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) {
if (!StringUtils.isEmpty(prestoConnectionParam.getOther())) {
return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther());
}
return prestoConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.sql.Connection;
@ -110,7 +110,7 @@ public class SparkDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) {
if (!StringUtils.isEmpty(sparkConnectionParam.getOther())) {
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther());
}
return sparkConnectionParam.getJdbcUrl();

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java

@ -25,9 +25,9 @@ import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -82,7 +82,7 @@ public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) {
if (!StringUtils.isEmpty(sqlServerConnectionParam.getOther())) {
return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther());
}
return sqlServerConnectionParam.getJdbcUrl();

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java

@ -24,7 +24,8 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.HashMap;
import java.util.List;
@ -245,7 +246,7 @@ public class TaskNode {
}
public Boolean isForbidden() {
return (StringUtils.isNotEmpty(this.runFlag)
return (!StringUtils.isEmpty(this.runFlag)
&& this.runFlag.equals(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN));
}
@ -363,7 +364,7 @@ public class TaskNode {
* @return task time out parameter
*/
public TaskTimeoutParameter getTaskTimeoutParameter() {
if (StringUtils.isNotEmpty(this.getTimeout())) {
if (!StringUtils.isEmpty(this.getTimeout())) {
String formatStr = String.format("%s,%s", TaskTimeoutStrategy.WARN.name(), TaskTimeoutStrategy.FAILED.name());
String taskTimeout = this.getTimeout().replace(formatStr, TaskTimeoutStrategy.WARNFAILED.name());
return JSONUtils.parseObject(taskTimeout, TaskTimeoutParameter.class);

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java

@ -21,7 +21,8 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
@ -168,7 +169,7 @@ public abstract class AbstractParameters implements IParameters {
String[] formatResult = result.split("\\$VarPool\\$");
Map<String, String> format = new HashMap<>();
for (String info : formatResult) {
if (StringUtils.isNotEmpty(info) && info.contains("=")) {
if (!StringUtils.isEmpty(info) && info.contains("=")) {
String[] keyValue = info.split("=");
format.put(keyValue[0], keyValue[1]);
}

9
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/datax/DataxParameters.java

@ -20,7 +20,8 @@ package org.apache.dolphinscheduler.common.task.datax;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
@ -217,10 +218,10 @@ public class DataxParameters extends AbstractParameters {
if (customConfig == Flag.NO.ordinal()) {
return dataSource != 0
&& dataTarget != 0
&& StringUtils.isNotEmpty(sql)
&& StringUtils.isNotEmpty(targetTable);
&& !StringUtils.isEmpty(sql)
&& !StringUtils.isEmpty(targetTable);
} else {
return StringUtils.isNotEmpty(json);
return !StringUtils.isEmpty(json);
}
}

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/http/HttpParameters.java

@ -22,7 +22,8 @@ import org.apache.dolphinscheduler.common.enums.HttpMethod;
import org.apache.dolphinscheduler.common.process.HttpProperty;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
@ -71,7 +72,7 @@ public class HttpParameters extends AbstractParameters {
@Override
public boolean checkParameters() {
return StringUtils.isNotEmpty(url);
return !StringUtils.isEmpty(url);
}
@Override

118
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/procedure/ProcedureParameters.java

@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.common.task.procedure;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
@ -29,62 +30,61 @@ import java.util.List;
*/
public class ProcedureParameters extends AbstractParameters {
/**
* data source typeeg MYSQL, POSTGRES, HIVE ...
*/
private String type;
/**
* data source id
*/
private int datasource;
/**
* procedure name
*/
private String method;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public int getDatasource() {
return datasource;
}
public void setDatasource(int datasource) {
this.datasource = datasource;
}
public String getMethod() {
return method;
}
public void setMethod(String method) {
this.method = method;
}
@Override
public boolean checkParameters() {
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(method);
}
@Override
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
@Override
public String toString() {
return "ProcessdureParam{" +
"type='" + type + '\'' +
", datasource=" + datasource +
", method='" + method + '\'' +
'}';
}
/**
* data source typeeg MYSQL, POSTGRES, HIVE ...
*/
private String type;
/**
* data source id
*/
private int datasource;
/**
* procedure name
*/
private String method;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public int getDatasource() {
return datasource;
}
public void setDatasource(int datasource) {
this.datasource = datasource;
}
public String getMethod() {
return method;
}
public void setMethod(String method) {
this.method = method;
}
@Override
public boolean checkParameters() {
return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(method);
}
@Override
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
@Override
public String toString() {
return "ProcessdureParam{"
+ "type='" + type + '\''
+ ", datasource=" + datasource
+ ", method='" + method + '\''
+ '}';
}
}

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java

@ -23,7 +23,8 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
@ -217,7 +218,7 @@ public class SqlParameters extends AbstractParameters {
@Override
public boolean checkParameters() {
return datasource != 0 && StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(sql);
return datasource != 0 && !StringUtils.isEmpty(type) && !StringUtils.isEmpty(sql);
}
@Override

25
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sqoop/SqoopParameters.java

@ -20,7 +20,8 @@ import org.apache.dolphinscheduler.common.enums.SqoopJobType;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
@ -28,7 +29,7 @@ import java.util.List;
/**
* sqoop parameters
*/
public class SqoopParameters extends AbstractParameters {
public class SqoopParameters extends AbstractParameters {
/**
* sqoop job type:
@ -180,17 +181,17 @@ public class SqoopParameters extends AbstractParameters {
}
if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isEmpty(customShell) &&
StringUtils.isNotEmpty(modelType) &&
StringUtils.isNotEmpty(jobName) &&
concurrency != 0 &&
StringUtils.isNotEmpty(sourceType) &&
StringUtils.isNotEmpty(targetType) &&
StringUtils.isNotEmpty(sourceParams) &&
StringUtils.isNotEmpty(targetParams);
sqoopParamsCheck = StringUtils.isEmpty(customShell)
&& !StringUtils.isEmpty(modelType)
&& !StringUtils.isEmpty(jobName)
&& concurrency != 0
&& !StringUtils.isEmpty(sourceType)
&& !StringUtils.isEmpty(targetType)
&& !StringUtils.isEmpty(sourceParams)
&& !StringUtils.isEmpty(targetParams);
} else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) {
sqoopParamsCheck = StringUtils.isNotEmpty(customShell) &&
StringUtils.isEmpty(jobName);
sqoopParamsCheck = !StringUtils.isEmpty(customShell)
&& StringUtils.isEmpty(jobName);
}
return sqoopParamsCheck;

33
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java

@ -1,33 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
public class BooleanUtils {
public static boolean isTrue(Boolean bool) {
if (bool == null) {
return false;
} else {
return bool;
}
}
public static boolean isNotTrue(Boolean bool) {
return !isTrue(bool);
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Collection;

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;

99
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.lang.StringUtils;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@ -26,16 +28,12 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.Date;
import java.util.Objects;
import java.util.TimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* date utils
*/
public class DateUtils {
public final class DateUtils {
static final long C0 = 1L;
static final long C1 = C0 * 1000L;
@ -45,36 +43,12 @@ public class DateUtils {
static final long C5 = C4 * 60L;
static final long C6 = C5 * 24L;
/**
* a default datetime formatter for the timestamp
*/
private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils");
}
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @return string formatted as yyyy-MM-dd HH:mm:ss
*/
public static String formatTimeStamp(long timeMillis) {
return formatTimeStamp(timeMillis, DEFAULT_DATETIME_FORMATTER);
}
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @param dateTimeFormatter expect formatter, like yyyy-MM-dd HH:mm:ss
* @return formatted string
*/
public static String formatTimeStamp(long timeMillis, DateTimeFormatter dateTimeFormatter) {
Objects.requireNonNull(dateTimeFormatter);
return dateTimeFormatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeMillis),
ZoneId.systemDefault()));
}
/**
* date to local datetime
*
@ -199,28 +173,6 @@ public class DateUtils {
return Math.abs(d1.getTime() - d2.getTime());
}
/**
* get hours between two dates
*
* @param d1 date1
* @param d2 date2
* @return differ hours
*/
public static long diffHours(Date d1, Date d2) {
return (long) Math.ceil(diffMin(d1, d2) / 60.0);
}
/**
* get minutes between two dates
*
* @param d1 date1
* @param d2 date2
* @return differ minutes
*/
public static long diffMin(Date d1, Date d2) {
return (long) Math.ceil(differSec(d1, d2) / 60.0);
}
/**
* get the date of the specified date in the days before and after
*
@ -483,38 +435,6 @@ public class DateUtils {
Constants.YYYY_MM_DD_HH_MM_SS);
}
public static Date addYears(Date date, int amount) {
return add(date, 1, amount);
}
public static Date addMonths(Date date, int amount) {
return add(date, 2, amount);
}
public static Date addWeeks(Date date, int amount) {
return add(date, 3, amount);
}
public static Date addDays(Date date, int amount) {
return add(date, 5, amount);
}
public static Date addHours(Date date, int amount) {
return add(date, 11, amount);
}
public static Date addMinutes(Date date, int amount) {
return add(date, 12, amount);
}
public static Date addSeconds(Date date, int amount) {
return add(date, 13, amount);
}
public static Date addMilliseconds(Date date, int amount) {
return add(date, 14, amount);
}
/**
* get date
*
@ -590,19 +510,6 @@ public class DateUtils {
* Time unit representing one thousandth of a second
*/
public static class MILLISECONDS {
public static long toSeconds(long d) {
return d / (C3 / C2);
}
public static long toMinutes(long d) {
return d / (C4 / C2);
}
public static long toHours(long d) {
return d / (C5 / C2);
}
public static long toDays(long d) {
return d / (C6 / C2);
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
/**
* encryption utils

185
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@ -22,8 +22,8 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@ -32,13 +32,9 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional;
import org.slf4j.Logger;
@ -68,7 +64,7 @@ public class FileUtils {
public static String suffix(String filename) {
String fileSuffix = "";
if (StringUtils.isNotEmpty(filename)) {
if (!StringUtils.isEmpty(filename)) {
int lastIndex = filename.lastIndexOf('.');
if (lastIndex > 0) {
fileSuffix = filename.substring(lastIndex + 1);
@ -168,7 +164,6 @@ public class FileUtils {
* @return true if write success
*/
public static boolean writeContent2File(String content, String filePath) {
boolean flag = true;
BufferedReader bufferedReader = null;
BufferedWriter bufferedWriter = null;
try {
@ -189,184 +184,12 @@ public class FileUtils {
bufferedWriter.close();
} catch (IOException e) {
FileUtils.logger.error(e.getMessage(), e);
flag = false;
return flag;
return false;
} finally {
IOUtils.closeQuietly(bufferedWriter);
IOUtils.closeQuietly(bufferedReader);
}
return flag;
}
/**
* Writes a String to a file creating the file if it does not exist.
* <p>
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
* @since 2.4
*/
public static void writeStringToFile(File file, String data, Charset encoding) throws IOException {
writeStringToFile(file, data, encoding, false);
}
/**
* Writes a String to a file creating the file if it does not exist.
* <p>
* NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
*/
public static void writeStringToFile(File file, String data, String encoding) throws IOException {
writeStringToFile(file, data, encoding, false);
}
/**
* Writes a String to a file creating the file if it does not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @since 2.3
*/
public static void writeStringToFile(File file, String data, Charset encoding, boolean append) throws IOException {
OutputStream out = null;
try {
out = openOutputStream(file, append);
IOUtils.write(data, out, encoding);
out.close(); // don't swallow close Exception if copy completes normally
} finally {
IOUtils.closeQuietly(out);
}
}
/**
* Writes a String to a file creating the file if it does not exist.
*
* @param file the file to write
* @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* supported by the VM
* @since 2.1
*/
public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException {
writeStringToFile(file, data, Charsets.toCharset(encoding), append);
}
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
* @param file the file to write
* @param data the content to write to the file
* @throws IOException in case of an I/O error
*/
public static void writeStringToFile(File file, String data) throws IOException {
writeStringToFile(file, data, Charset.defaultCharset(), false);
}
/**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
*
* @param file the file to write
* @param data the content to write to the file
* @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting
* @throws IOException in case of an I/O error
* @since 2.1
*/
public static void writeStringToFile(File file, String data, boolean append) throws IOException {
writeStringToFile(file, data, Charset.defaultCharset(), append);
}
/**
* Opens a {@link FileOutputStream} for the specified file, checking and
* creating the parent directory if it does not exist.
* <p>
* At the end of the method either the stream will be successfully opened,
* or an exception will have been thrown.
* <p>
* The parent directory will be created if it does not exist.
* The file will be created if it does not exist.
* An exception is thrown if the file object exists but is a directory.
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
* @param file the file to open for output, must not be {@code null}
* @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to
* @throws IOException if a parent directory needs creating but that fails
* @since 1.3
*/
public static FileOutputStream openOutputStream(File file) throws IOException {
return openOutputStream(file, false);
}
/**
* Opens a {@link FileOutputStream} for the specified file, checking and
* creating the parent directory if it does not exist.
* <p>
* At the end of the method either the stream will be successfully opened,
* or an exception will have been thrown.
* <p>
* The parent directory will be created if it does not exist.
* The file will be created if it does not exist.
* An exception is thrown if the file object exists but is a directory.
* An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created.
*
* @param file the file to open for output, must not be {@code null}
* @param append if {@code true}, then bytes will be added to the
* end of the file rather than overwriting
* @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to
* @throws IOException if a parent directory needs creating but that fails
* @since 2.1
*/
public static FileOutputStream openOutputStream(File file, boolean append) throws IOException {
if (file.exists()) {
if (file.isDirectory()) {
throw new IOException("File '" + file + "' exists but is a directory");
}
if (!file.canWrite()) {
throw new IOException("File '" + file + "' cannot be written to");
}
} else {
File parent = file.getParentFile();
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
throw new IOException("Directory '" + parent + "' could not be created");
}
}
return new FileOutputStream(file, append);
}
/**
* deletes a directory recursively
*
* @param dir directory
* @throws IOException in case deletion is unsuccessful
*/
public static void deleteDir(String dir) throws IOException {
org.apache.commons.io.FileUtils.deleteDirectory(new File(dir));
return true;
}
/**

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.exception.BaseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java

@ -24,6 +24,8 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKN
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
@ -206,7 +208,7 @@ public class OSUtils {
*/
private static List<String> getUserListFromMac() throws IOException {
String result = exeCmd("dscl . list /users");
if (StringUtils.isNotEmpty(result)) {
if (!StringUtils.isEmpty(result)) {
return Arrays.asList(result.split("\n"));
}
@ -392,7 +394,7 @@ public class OSUtils {
}
} else {
String result = exeCmd("groups");
if (StringUtils.isNotEmpty(result)) {
if (!StringUtils.isEmpty(result)) {
String[] groupInfo = result.split(" ");
return groupInfo[0];
}

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -25,6 +25,8 @@ import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import org.apache.commons.lang.StringUtils;
import java.sql.PreparedStatement;
import java.util.Date;
import java.util.HashMap;
@ -99,7 +101,7 @@ public class ParameterUtils {
String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) {
if (!StringUtils.isEmpty(cronTimeStr)) {
cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
} else {
@ -209,7 +211,7 @@ public class ParameterUtils {
*/
public static String handleEscapes(String inputString) {
if (StringUtils.isNotEmpty(inputString)) {
if (!StringUtils.isEmpty(inputString)) {
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
}
return inputString;

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java

@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.commons.lang.StringUtils;
/**
* heartbeat for ZK reigster res info
*/
@ -113,7 +115,7 @@ public class ResInfo {
* @return heartbeat info is valid
*/
public static boolean isValidHeartbeatForRegistryInfo(String heartBeatInfo) {
if (StringUtils.isNotEmpty(heartBeatInfo)) {
if (!StringUtils.isEmpty(heartBeatInfo)) {
String[] parts = heartBeatInfo.split(Constants.COMMA);
return parts.length == Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH
|| parts.length == Constants.HEARTBEAT_WITH_WEIGHT_FOR_ZOOKEEPER_INFO_LENGTH;

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;

316
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java

@ -1,12 +1,12 @@
/**
/*
* Copyright 2004-2020 the original author or authors.
*
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -32,113 +32,85 @@ import org.slf4j.LoggerFactory;
*/
public class ScriptRunner {
public static final Logger logger = LoggerFactory.getLogger(ScriptRunner.class);
private static final String DEFAULT_DELIMITER = ";";
private Connection connection;
private boolean stopOnError;
private boolean autoCommit;
private String delimiter = DEFAULT_DELIMITER;
private boolean fullLineDelimiter = false;
public ScriptRunner(Connection connection, boolean autoCommit, boolean stopOnError) {
this.connection = connection;
this.autoCommit = autoCommit;
this.stopOnError = stopOnError;
}
public void setDelimiter(String delimiter, boolean fullLineDelimiter) {
this.delimiter = delimiter;
this.fullLineDelimiter = fullLineDelimiter;
}
/**
* Runs an SQL script (read in using the Reader parameter)
*
* @param reader
* - the source of the script
* @throws IOException errors
* @throws SQLException errors
*/
public void runScript(Reader reader) throws IOException, SQLException {
try {
boolean originalAutoCommit = connection.getAutoCommit();
try {
if (originalAutoCommit != this.autoCommit) {
connection.setAutoCommit(this.autoCommit);
}
runScript(connection, reader);
} finally {
connection.setAutoCommit(originalAutoCommit);
}
} catch (IOException | SQLException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException("Error running script. Cause: " + e, e);
}
}
public void runScript(Reader reader, String dbName) throws IOException, SQLException {
try {
boolean originalAutoCommit = connection.getAutoCommit();
try {
if (originalAutoCommit != this.autoCommit) {
connection.setAutoCommit(this.autoCommit);
}
runScript(connection, reader, dbName);
} finally {
connection.setAutoCommit(originalAutoCommit);
}
} catch (IOException | SQLException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException("Error running script. Cause: " + e, e);
}
}
/**
* Runs an SQL script (read in using the Reader parameter) using the connection
* passed in
*
* @param conn
* - the connection to use for the script
* @param reader
* - the source of the script
* @throws SQLException
* if any SQL errors occur
* @throws IOException
* if there is an error reading from the Reader
*/
private void runScript(Connection conn, Reader reader) throws IOException, SQLException {
StringBuffer command = null;
try {
LineNumberReader lineReader = new LineNumberReader(reader);
String line = null;
while ((line = lineReader.readLine()) != null) {
if (command == null) {
command = new StringBuffer();
}
String trimmedLine = line.trim();
if (trimmedLine.startsWith("--")) {
logger.info(trimmedLine);
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("//")) {
// Do nothing
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("--")) {
// Do nothing
} else if (trimmedLine.startsWith("delimiter")) {
String newDelimiter = trimmedLine.split(" ")[1];
this.setDelimiter(newDelimiter, fullLineDelimiter);
} else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter())
|| fullLineDelimiter && trimmedLine.equals(getDelimiter())) {
command.append(line.substring(0, line.lastIndexOf(getDelimiter())));
command.append(" ");
public static final Logger logger = LoggerFactory.getLogger(ScriptRunner.class);
private static final String DEFAULT_DELIMITER = ";";
private final Connection connection;
private final boolean stopOnError;
private final boolean autoCommit;
private String delimiter = DEFAULT_DELIMITER;
private boolean fullLineDelimiter = false;
public ScriptRunner(Connection connection, boolean autoCommit, boolean stopOnError) {
this.connection = connection;
this.autoCommit = autoCommit;
this.stopOnError = stopOnError;
}
public void setDelimiter(String delimiter, boolean fullLineDelimiter) {
this.delimiter = delimiter;
this.fullLineDelimiter = fullLineDelimiter;
}
/**
* Runs an SQL script (read in using the Reader parameter)
*
* @param reader - the source of the script
* @throws IOException errors
* @throws SQLException errors
*/
public void runScript(Reader reader) throws IOException, SQLException {
try {
boolean originalAutoCommit = connection.getAutoCommit();
try {
if (originalAutoCommit != this.autoCommit) {
connection.setAutoCommit(this.autoCommit);
}
runScript(connection, reader);
} finally {
connection.setAutoCommit(originalAutoCommit);
}
} catch (IOException | SQLException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException("Error running script. Cause: " + e, e);
}
}
/**
* Runs an SQL script (read in using the Reader parameter) using the connection
* passed in
*
* @param conn - the connection to use for the script
* @param reader - the source of the script
* @throws SQLException if any SQL errors occur
* @throws IOException if there is an error reading from the Reader
*/
private void runScript(Connection conn, Reader reader) throws IOException, SQLException {
StringBuffer command = null;
try {
LineNumberReader lineReader = new LineNumberReader(reader);
String line;
while ((line = lineReader.readLine()) != null) {
if (command == null) {
command = new StringBuffer();
}
String trimmedLine = line.trim();
if (trimmedLine.startsWith("--")) {
logger.info(trimmedLine);
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("//")) {
// Do nothing
} else if (trimmedLine.startsWith("delimiter")) {
String newDelimiter = trimmedLine.split(" ")[1];
this.setDelimiter(newDelimiter, fullLineDelimiter);
} else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter())
|| fullLineDelimiter && trimmedLine.equals(getDelimiter())) {
command.append(line, 0, line.lastIndexOf(getDelimiter()));
command.append(" ");
logger.info("sql: {}", command);
try (Statement statement = conn.createStatement()) {
@ -160,104 +132,32 @@ public class ScriptRunner {
logger.info("");
}
}
}
} catch (SQLException e) {
logger.error("SQLException", e);
throw e;
}
command = null;
Thread.yield();
} else {
command.append(line);
command.append(" ");
}
}
} catch (SQLException e) {
logger.error("Error executing: {}", command);
throw e;
} catch (IOException e) {
e.fillInStackTrace();
logger.error("Error executing: {}", command);
throw e;
}
}
private void runScript(Connection conn, Reader reader , String dbName) throws IOException, SQLException {
StringBuffer command = null;
String sql = "";
String appKey = dbName.substring(dbName.lastIndexOf("_")+1, dbName.length());
try {
LineNumberReader lineReader = new LineNumberReader(reader);
String line = null;
while ((line = lineReader.readLine()) != null) {
if (command == null) {
command = new StringBuffer();
}
String trimmedLine = line.trim();
if (trimmedLine.startsWith("--")) {
logger.info(trimmedLine);
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("//")) {
// Do nothing
} else if (trimmedLine.length() < 1 || trimmedLine.startsWith("--")) {
// Do nothing
} else if (trimmedLine.startsWith("delimiter")) {
String newDelimiter = trimmedLine.split(" ")[1];
this.setDelimiter(newDelimiter, fullLineDelimiter);
} else if (!fullLineDelimiter && trimmedLine.endsWith(getDelimiter())
|| fullLineDelimiter && trimmedLine.equals(getDelimiter())) {
command.append(line.substring(0, line.lastIndexOf(getDelimiter())));
command.append(" ");
sql = command.toString().replaceAll("\\{\\{APPDB\\}\\}", dbName);
logger.info("sql : {}", sql);
try (Statement statement = conn.createStatement()) {
statement.execute(sql);
try (ResultSet rs = statement.getResultSet()) {
if (stopOnError && rs != null) {
ResultSetMetaData md = rs.getMetaData();
int cols = md.getColumnCount();
for (int i = 0; i < cols; i++) {
String name = md.getColumnLabel(i);
logger.info("{} \t", name);
}
logger.info("");
while (rs.next()) {
for (int i = 0; i < cols; i++) {
String value = rs.getString(i);
logger.info("{} \t", value);
}
logger.info("");
}
}
}
}
} catch (SQLException e) {
logger.error("SQLException", e);
throw e;
}
command = null;
Thread.yield();
} else {
command.append(line);
command.append(" ");
}
}
} catch (SQLException e) {
throw e;
} catch (IOException e) {
e.fillInStackTrace();
logger.error("Error executing: {}", sql);
throw e;
}
}
private String getDelimiter() {
return delimiter;
}
}
command = null;
Thread.yield();
} else {
command.append(line);
command.append(" ");
}
}
} catch (SQLException e) {
logger.error("Error executing: {}", command);
throw e;
} catch (IOException e) {
e.fillInStackTrace();
logger.error("Error executing: {}", command);
throw e;
}
}
private String getDelimiter() {
return delimiter;
}
}

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java

@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.lang.StringUtils;
/**
* sensitive log Util
*/
@ -34,7 +36,7 @@ public class SensitiveLogUtils {
*/
public static String maskDataSourcePwd(String dataSourcePwd) {
if (StringUtils.isNotEmpty(dataSourcePwd)) {
if (!StringUtils.isEmpty(dataSourcePwd)) {
dataSourcePwd = Constants.PASSWORD_DEFAULT;
}
return dataSourcePwd;

226
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -1,226 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.Collection;
import java.util.Iterator;
/**
* java.lang.String utils class
*/
public class StringUtils {
/**
* The empty String {@code ""}.
*/
public static final String EMPTY = "";
public static final int INDEX_NOT_FOUND = -1;
private StringUtils() {
throw new UnsupportedOperationException("Construct StringUtils");
}
/**
* <p>Checks if a CharSequence is empty ("") or null.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is empty or null
*/
public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
/**
* <p>Checks if a CharSequence is not empty ("") and not null.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is not empty and not null
*/
public static boolean isNotEmpty(final CharSequence cs) {
return !isEmpty(cs);
}
/**
* <p>Checks if a CharSequence is empty (""), null or whitespace only.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is null, empty or whitespace only
*/
public static boolean isBlank(final CharSequence cs) {
int strLen;
if (cs == null || (strLen = cs.length()) == 0) {
return true;
}
for (int i = 0; i < strLen; i++) {
if (!Character.isWhitespace(cs.charAt(i))) {
return false;
}
}
return true;
}
/**
* <p>Checks if a CharSequence is not empty (""), not null and not whitespace only.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is not empty and not null and not whitespace only
*/
public static boolean isNotBlank(final CharSequence cs) {
return !isBlank(cs);
}
/**
* <p>Replace all strings matching the regular expression \t \n \r with _</p>
*
* @param src the String , may be null
* @return the string that has been replaced
*/
public static String replaceNRTtoUnderline(String src) {
return isBlank(src) ? src : src.replaceAll("[\n|\r|\t]", "_");
}
/**
* <p>Removes control characters (char &lt;= 32) from both
* ends of this String, handling {@code null} by returning
* {@code null}.</p>
*
* @param str the String to be trimmed, may be null
* @return the trimmed string, {@code null} if null String input
*/
public static String trim(final String str) {
return str == null ? null : str.trim();
}
/**
* <p>Returns either the passed in CharSequence, or if the CharSequence is
* whitespace, empty ("") or {@code null}, the value of {@code defaultStr}.</p>
*
* @param <T> the specific kind of CharSequence
* @param str the CharSequence to check, may be null
* @param defaultStr the default CharSequence to return
* if the input is whitespace, empty ("") or {@code null}, may be null
* @return the passed in CharSequence, or the default
*/
public static <T extends CharSequence> T defaultIfBlank(final T str, final T defaultStr) {
return isBlank(str) ? defaultStr : str;
}
/**
* <p>Compares two String, returning {@code true} if they represent
* equal string, ignoring case.</p>
*
* @param str1 the first String, may be null
* @param str2 the second String, may be null
* @return {@code true} if the String are equal, case insensitive, or
* both {@code null}
*/
public static boolean equalsIgnoreCase(String str1, String str2) {
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
}
public static String substringBefore(final String str, final String separator) {
if (isEmpty(str) || separator == null) {
return str;
}
if (separator.isEmpty()) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return str;
}
return str.substring(0, pos);
}
public static String substringAfter(final String str, final String separator) {
if (isEmpty(str)) {
return str;
}
if (separator == null) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return EMPTY;
}
return str.substring(pos + separator.length());
}
public static long strDigitToLong(String str, long defaultValue) {
if (str == null) {
return defaultValue;
} else {
try {
return Long.parseLong(str);
} catch (NumberFormatException var4) {
return defaultValue;
}
}
}
/**
* <p>Joins the elements of the provided Collection into a single String
* containing the provided Collection of elements.</p>
*
* @param collection the collection, may be null
* @param separator the separator
* @return a single String
*/
public static String join(Collection<?> collection, String separator) {
return collection == null ? null : join(collection.iterator(), separator);
}
/**
* <p>Joins the elements of the provided Iterator into a single String
* containing the provided Iterator of elements.</p>
*
* @param iterator the iterator, may be null
* @param separator the separator
* @return a single String
*/
public static String join(Iterator<?> iterator, String separator) {
if (iterator == null) {
return null;
} else if (!iterator.hasNext()) {
return "";
} else {
Object first = iterator.next();
if (!iterator.hasNext()) {
return first == null ? "" : first.toString();
} else {
StringBuilder buf = new StringBuilder(256);
if (first != null) {
buf.append(first);
}
while (iterator.hasNext()) {
if (separator != null) {
buf.append(separator);
}
Object obj = iterator.next();
if (obj != null) {
buf.append(obj);
}
}
return buf.toString();
}
}
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java

@ -33,7 +33,8 @@ import static org.apache.dolphinscheduler.common.Constants.SUBTRACT_STRING;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.AbstractMap;
import java.util.ArrayList;

26
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java

@ -42,20 +42,6 @@ public class DateUtilsTest {
Assert.assertEquals("01 09:23:08", readableDate);
}
@Test
public void testConvertTimeStampsToString() {
TimeZone defaultTimeZone = TimeZone.getDefault();
final TimeZone timeZone = TimeZone.getTimeZone("Asia/Shanghai");
TimeZone.setDefault(timeZone);
long timeMillis = 1625989249021L;
Assert.assertEquals("2021-07-11 15:40:49", DateUtils.formatTimeStamp(timeMillis));
DateTimeFormatter testFormatter = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
Assert.assertEquals("2021/07/11 15:40:49", DateUtils.formatTimeStamp(timeMillis, testFormatter));
TimeZone.setDefault(defaultTimeZone);
}
@Test
public void testWeek() {
Date curr = DateUtils.stringToDate("2019-02-01 00:00:00");
@ -69,18 +55,6 @@ public class DateUtilsTest {
}
@Test
public void diffHours() {
Date d1 = DateUtils.stringToDate("2019-01-28 00:00:00");
Date d2 = DateUtils.stringToDate("2019-01-28 20:00:00");
Assert.assertEquals(DateUtils.diffHours(d1, d2), 20);
Date d3 = DateUtils.stringToDate("2019-01-28 20:00:00");
Assert.assertEquals(DateUtils.diffHours(d3, d2), 0);
Assert.assertEquals(DateUtils.diffHours(d2, d1), 20);
Date d4 = null;
Assert.assertEquals(DateUtils.diffHours(d2, d4), 0);
}
@Test
public void dateToString() {
Date d1 = DateUtils.stringToDate("2019-01-28");

39
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java

@ -15,11 +15,11 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.io.StringReader;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.StringReader;
import java.sql.*;
public class ScriptRunnerTest {
@Test
@ -33,38 +33,5 @@ public class ScriptRunnerTest {
exception = e;
}
Assert.assertNotNull(exception);
//connect is not null
runScript("");
}
private void runScript(String dbName) {
try {
Connection conn = Mockito.mock(Connection.class);
Mockito.when(conn.getAutoCommit()).thenReturn(true);
PreparedStatement st = Mockito.mock(PreparedStatement.class);
Mockito.when(conn.createStatement()).thenReturn(st);
ResultSet rs = Mockito.mock(ResultSet.class);
Mockito.when(st.getResultSet()).thenReturn(rs);
ResultSetMetaData md = Mockito.mock(ResultSetMetaData.class);
Mockito.when(rs.getMetaData()).thenReturn(md);
Mockito.when(md.getColumnCount()).thenReturn(2);
Mockito.when(rs.next()).thenReturn(true, false);
ScriptRunner s = new ScriptRunner(conn, true, true);
if (dbName.isEmpty()) {
s.runScript(new StringReader("select 1;"));
} else {
s.runScript(new StringReader("select 1;"), dbName);
}
Mockito.verify(md).getColumnLabel(1);
} catch(Exception e) {
Assert.assertNotNull(e);
}
}
@Test
public void testRunScriptWithDbName() {
//connect is not null
runScript("db_test");
}
}

96
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java

@ -1,96 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
public class StringUtilsTest {
@Test
public void testIsNotEmpty() {
//null string
boolean b = StringUtils.isNotEmpty(null);
Assert.assertFalse(b);
//"" string
b = StringUtils.isNotEmpty("");
Assert.assertFalse(b);
//" " string
b = StringUtils.isNotEmpty(" ");
Assert.assertTrue(b);
//"test" string
b = StringUtils.isNotEmpty("test");
Assert.assertTrue(b);
}
@Test
public void testIsNotBlank() {
//null string
boolean b = StringUtils.isNotBlank(null);
Assert.assertFalse(b);
//"" string
b = StringUtils.isNotBlank("");
Assert.assertFalse(b);
//" " string
b = StringUtils.isNotBlank(" ");
Assert.assertFalse(b);
//" test " string
b = StringUtils.isNotBlank(" test ");
Assert.assertTrue(b);
//"test" string
b = StringUtils.isNotBlank("test");
Assert.assertTrue(b);
}
@Test
public void testTrim() {
String trim = StringUtils.trim(null);
Assert.assertNull(trim);
trim = StringUtils.trim(" test ");
Assert.assertEquals("test", trim);
}
@Test
public void testDefaultIfBlank() {
String defaultStr = StringUtils.defaultIfBlank("", "defaultStr");
Assert.assertEquals("defaultStr", defaultStr);
defaultStr = StringUtils.defaultIfBlank("test", "defaultStr");
Assert.assertEquals("test", defaultStr);
}
@Test
public void testJoin() {
List<String> list = new ArrayList<>();
list.add("1");
list.add("3");
list.add("4");
String join = StringUtils.join(list, ",");
Assert.assertEquals("1,3,4", join);
}
}

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.enums.AlertEvent;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
import org.apache.dolphinscheduler.common.enums.AlertWarnLevel;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
@ -33,6 +32,8 @@ import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;

9
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UdfFunc.java

@ -19,15 +19,18 @@ package org.apache.dolphinscheduler.dao.entity;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.Date;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.KeyDeserializer;
import java.io.IOException;
import java.util.Date;
/**
* udf function

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@ -23,6 +23,9 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.AbstractBaseDao;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -144,7 +147,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
// Execute the dolphinscheduler_dml.sql script to import related data of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, false, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
Reader initSqlReader = new FileReader(mysqlSQLFilePath);
initScriptRunner.runScript(initSqlReader);
conn.commit();
@ -189,7 +192,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
conn = dataSource.getConnection();
// Execute the dolphinscheduler_ddl.sql script to create the table structure of dolphinscheduler
ScriptRunner initScriptRunner = new ScriptRunner(conn, true, true);
Reader initSqlReader = new FileReader(new File(mysqlSQLFilePath));
Reader initSqlReader = new FileReader(mysqlSQLFilePath);
initScriptRunner.runScript(initSqlReader);
} catch (IOException e) {

7
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java

@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.utils.BooleanUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import java.util.Date;
@ -133,7 +132,7 @@ public class QueueMapperTest {
Assert.assertNull(queueMapper.existQueue("queue", null));
Assert.assertNull(queueMapper.existQueue(null, "queue"));
Queue queue = insertOne();
Assert.assertTrue(BooleanUtils.isTrue(queueMapper.existQueue(queue.getQueue(), null)));
Assert.assertTrue(BooleanUtils.isTrue(queueMapper.existQueue(null, queue.getQueueName())));
Assert.assertTrue(queueMapper.existQueue(queue.getQueue(), null) == Boolean.TRUE);
Assert.assertTrue(queueMapper.existQueue(null, queue.getQueueName()) == Boolean.TRUE);
}
}
}

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java

@ -22,7 +22,8 @@ import ch.qos.logback.classic.spi.ILoggingEvent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -57,7 +58,7 @@ public class SensitiveDataConverter extends MessageConverter {
String tempLogMsg = oriLogMsg;
if (StringUtils.isNotEmpty(tempLogMsg)) {
if (!StringUtils.isEmpty(tempLogMsg)) {
tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
}
return tempLogMsg;

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java

@ -35,7 +35,6 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EnumUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.Resource;
@ -57,6 +56,8 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.TaskPriority;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -334,7 +335,7 @@ public class TaskPriorityQueueConsumer extends Thread {
// whether udf type
boolean udfTypeFlag = EnumUtils.isValidEnum(UdfType.class, sqlParameters.getType())
&& StringUtils.isNotEmpty(sqlParameters.getUdfs());
&& !StringUtils.isEmpty(sqlParameters.getUdfs());
if (udfTypeFlag) {
String[] udfFunIds = sqlParameters.getUdfs().split(",");

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java

@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.server.master.dispatch;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
@ -27,6 +26,8 @@ import org.apache.dolphinscheduler.server.master.dispatch.executor.ExecutorManag
import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
import org.apache.dolphinscheduler.server.master.dispatch.host.HostManager;
import org.apache.commons.lang.StringUtils;
import java.util.concurrent.ConcurrentHashMap;
import org.springframework.beans.factory.InitializingBean;

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java

@ -20,13 +20,14 @@ package org.apache.dolphinscheduler.server.master.dispatch.host;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -88,7 +89,7 @@ public abstract class CommonHostManager implements HostManager {
protected int getWorkerHostWeightFromHeartbeat(String heartbeat) {
int hostWeight = Constants.DEFAULT_WORKER_HOST_WEIGHT;
if (StringUtils.isNotEmpty(heartbeat)) {
if (!StringUtils.isEmpty(heartbeat)) {
String[] parts = heartbeat.split(Constants.COMMA);
if (ResInfo.isNewHeartbeatWithWeight(parts)) {
hostWeight = Integer.parseInt(parts[10]);

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java

@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
@ -46,6 +45,8 @@ import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.dolphinscheduler.spi.register.RegistryConnectListener;
import org.apache.dolphinscheduler.spi.register.RegistryConnectState;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@ -160,7 +161,7 @@ public class MasterRegistryClient {
registryClient.getLock(failoverPath);
String serverHost = null;
if (StringUtils.isNotEmpty(path)) {
if (!StringUtils.isEmpty(path)) {
serverHost = registryClient.getHostByEventDataPath(path);
if (StringUtils.isEmpty(serverHost)) {
logger.error("server down error: unknown path: {}", path);

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
@ -35,6 +34,7 @@ import org.apache.dolphinscheduler.spi.register.DataChangeEvent;
import org.apache.dolphinscheduler.spi.register.SubscribeListener;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Collections;

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java

@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.common.enums.StateEventType;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
@ -32,6 +31,8 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;

69
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java

@ -47,9 +47,7 @@ import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@ -70,6 +68,8 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -182,13 +182,6 @@ public class WorkflowExecuteThread implements Runnable {
*/
private NettyExecutorManager nettyExecutorManager;
/**
* submit post node
*
* @param parentNodeName parent node name
*/
private Map<String, Object> propToValue = new ConcurrentHashMap<>();
private ConcurrentLinkedQueue<StateEvent> stateEvents = new ConcurrentLinkedQueue<>();
private List<Date> complementListDate = Lists.newLinkedList();
@ -474,21 +467,6 @@ public class WorkflowExecuteThread implements Runnable {
}
}
/**
* prepare process parameter
*
* @throws Exception exception
*/
private void prepareProcess() throws Exception {
// gen process dag
buildFlowDag();
// init task queue
initTaskQueue();
logger.info("prepare process :{} end", processInstance.getId());
}
/**
* process end handle
*/
@ -1053,23 +1031,6 @@ public class WorkflowExecuteThread implements Runnable {
return state;
}
/**
* whether standby task list have retry tasks
*/
private boolean retryTaskExists() {
boolean result = false;
for (Iterator<TaskInstance> iter = readyToSubmitTaskQueue.iterator(); iter.hasNext(); ) {
TaskInstance task = iter.next();
if (task.getState().typeIsFailure()) {
result = true;
break;
}
}
return result;
}
/**
* whether complement end
*
@ -1171,32 +1132,6 @@ public class WorkflowExecuteThread implements Runnable {
return false;
}
/**
* whether check process time out
*
* @param processInstance task instance
* @return true if time out of process instance > running time of process instance
*/
private boolean checkProcessTimeOut(ProcessInstance processInstance) {
if (processInstance.getTimeout() == 0) {
return false;
}
Date now = new Date();
long runningTime = DateUtils.diffMin(now, processInstance.getStartTime());
return runningTime > processInstance.getTimeout();
}
/**
* whether can submit task to queue
*
* @return boolean
*/
private boolean canSubmitTaskToQueue() {
return OSUtils.checkResource(masterConfig.getMasterMaxCpuloadAvg(), masterConfig.getMasterReservedMemory());
}
/**
* close the on going tasks
*/

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java

@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master.runner.task;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand;
@ -36,6 +35,8 @@ import org.apache.dolphinscheduler.service.queue.TaskPriority;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import org.slf4j.Logger;

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/SwitchTaskProcessor.java

@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters;
import org.apache.dolphinscheduler.common.task.switchtask.SwitchResultVo;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@ -35,6 +34,8 @@ import org.apache.dolphinscheduler.server.utils.SwitchTaskUtils;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.HashMap;
import java.util.List;

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java

@ -17,7 +17,9 @@
package org.apache.dolphinscheduler.server.monitor;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RunConfig.java

@ -17,7 +17,9 @@
package org.apache.dolphinscheduler.server.monitor;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
@ -95,4 +97,4 @@ public class RunConfig {
public void setSshPort(String sshPort) {
this.sshPort = sshPort;
}
}
}

300
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java

@ -26,11 +26,12 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@ -46,11 +47,7 @@ import org.slf4j.LoggerFactory;
* mainly used to get the start command line of a process.
*/
public class ProcessUtils {
/**
* logger
*/
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
/**
* Initialization regularization, solve the problem of pre-compilation performance,
@ -63,233 +60,6 @@ public class ProcessUtils {
*/
private static final Pattern WINDOWSATTERN = Pattern.compile("\\w+\\((\\d+)\\)");
private static final String LOCAL_PROCESS_EXEC = "jdk.lang.Process.allowAmbiguousCommands";
/**
* build command line characters.
*
* @param commandList command list
* @return command
*/
public static String buildCommandStr(List<String> commandList) {
String cmdstr;
String[] cmd = commandList.toArray(new String[0]);
SecurityManager security = System.getSecurityManager();
boolean allowAmbiguousCommands = isAllowAmbiguousCommands(security);
if (allowAmbiguousCommands) {
String executablePath = new File(cmd[0]).getPath();
if (needsEscaping(VERIFICATION_LEGACY, executablePath)) {
executablePath = quoteString(executablePath);
}
cmdstr = createCommandLine(
VERIFICATION_LEGACY, executablePath, cmd);
} else {
String executablePath;
try {
executablePath = getExecutablePath(cmd[0]);
} catch (IllegalArgumentException e) {
StringBuilder join = new StringBuilder();
for (String s : cmd) {
join.append(s).append(' ');
}
cmd = getTokensFromCommand(join.toString());
executablePath = getExecutablePath(cmd[0]);
// Check new executable name once more
if (security != null) {
security.checkExec(executablePath);
}
}
cmdstr = createCommandLine(
isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd);
}
return cmdstr;
}
/**
* check is allow ambiguous commands
*
* @param security security manager
* @return allow ambiguous command flag
*/
private static boolean isAllowAmbiguousCommands(SecurityManager security) {
boolean allowAmbiguousCommands = false;
if (security == null) {
allowAmbiguousCommands = true;
String value = System.getProperty(LOCAL_PROCESS_EXEC);
if (value != null) {
allowAmbiguousCommands = !Constants.STRING_FALSE.equalsIgnoreCase(value);
}
}
return allowAmbiguousCommands;
}
/**
* get executable path.
*
* @param path path
* @return executable path
*/
private static String getExecutablePath(String path) {
boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments");
File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path);
return fileToRun.getPath();
}
/**
* whether is shell file.
*
* @param executablePath executable path
* @return true if endsWith .CMD or .BAT
*/
private static boolean isShellFile(String executablePath) {
String upPath = executablePath.toUpperCase();
return (upPath.endsWith(".CMD") || upPath.endsWith(".BAT"));
}
/**
* quote string.
*
* @param arg argument
* @return format arg
*/
private static String quoteString(String arg) {
return '"' + arg + '"';
}
/**
* get tokens from command.
*
* @param command command
* @return token string array
*/
private static String[] getTokensFromCommand(String command) {
ArrayList<String> matchList = new ArrayList<>(8);
Matcher regexMatcher = LazyPattern.PATTERN.matcher(command);
while (regexMatcher.find()) {
matchList.add(regexMatcher.group());
}
return matchList.toArray(new String[0]);
}
/**
* Lazy Pattern.
*/
private static class LazyPattern {
/**
* Escape-support version:
* "(\")((?:\\\\\\1|.)+?)\\1|([^\\s\"]+)";
*/
private static final Pattern PATTERN = Pattern.compile("[^\\s\"]+|\"[^\"]*\"");
}
/**
* verification cmd bat.
*/
private static final int VERIFICATION_CMD_BAT = 0;
/**
* verification win32.
*/
private static final int VERIFICATION_WIN32 = 1;
/**
* verification legacy.
*/
private static final int VERIFICATION_LEGACY = 2;
/**
* escape verification.
*/
private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'},
{' ', '\t', '<', '>'}, {' ', '\t'}};
/**
* create command line.
*
* @param verificationType verification type
* @param executablePath executable path
* @param cmd cmd
* @return command line
*/
private static String createCommandLine(int verificationType, final String executablePath, final String[] cmd) {
StringBuilder cmdbuf = new StringBuilder(80);
cmdbuf.append(executablePath);
for (int i = 1; i < cmd.length; ++i) {
cmdbuf.append(' ');
String s = cmd[i];
if (needsEscaping(verificationType, s)) {
cmdbuf.append('"').append(s);
if ((verificationType != VERIFICATION_CMD_BAT) && s.endsWith("\\")) {
cmdbuf.append('\\');
}
cmdbuf.append('"');
} else {
cmdbuf.append(s);
}
}
return cmdbuf.toString();
}
/**
* whether is quoted.
*
* @param noQuotesInside no quotes inside
* @param arg arg
* @param errorMessage error message
* @return boolean
*/
private static boolean isQuoted(boolean noQuotesInside, String arg, String errorMessage) {
int lastPos = arg.length() - 1;
if (lastPos >= 1 && arg.charAt(0) == '"' && arg.charAt(lastPos) == '"') {
// The argument has already been quoted.
if (noQuotesInside && arg.indexOf('"', 1) != lastPos) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
return true;
}
if (noQuotesInside && arg.indexOf('"') >= 0) {
// There is ["] inside.
throw new IllegalArgumentException(errorMessage);
}
return false;
}
/**
* whether needs escaping.
*
* @param verificationType verification type
* @param arg arg
* @return boolean
*/
private static boolean needsEscaping(int verificationType, String arg) {
boolean argIsQuoted = isQuoted((verificationType == VERIFICATION_CMD_BAT), arg, "Argument has embedded quote, use the explicit CMD.EXE call.");
if (!argIsQuoted) {
char[] testEscape = ESCAPE_VERIFICATION[verificationType];
for (char c : testEscape) {
if (arg.indexOf(c) >= 0) {
return true;
}
}
}
return false;
}
/**
* kill yarn application.
*
@ -299,21 +69,21 @@ public class ProcessUtils {
* @param executePath execute path
*/
public static void cancelApplication(List<String> appIds, Logger logger, String tenantCode, String executePath) {
if (CollectionUtils.isNotEmpty(appIds)) {
if (appIds == null || appIds.isEmpty()) {
return;
}
for (String appId : appIds) {
try {
ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
for (String appId : appIds) {
try {
ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
if (!applicationStatus.typeIsFinished()) {
String commandFile = String
.format("%s/%s.kill", executePath, appId);
String cmd = getKerberosInitCommand() + "yarn application -kill " + appId;
execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd);
}
} catch (Exception e) {
logger.error(String.format("Get yarn application app id [%s] status failed: [%s]", appId, e.getMessage()));
if (!applicationStatus.typeIsFinished()) {
String commandFile = String.format("%s/%s.kill", executePath, appId);
String cmd = getKerberosInitCommand() + "yarn application -kill " + appId;
execYarnKillCommand(logger, tenantCode, appId, commandFile, cmd);
}
} catch (Exception e) {
logger.error("Get yarn application app id [{}}] status failed", appId, e);
}
}
}
@ -321,15 +91,15 @@ public class ProcessUtils {
/**
* get kerberos init command
*/
public static String getKerberosInitCommand() {
static String getKerberosInitCommand() {
logger.info("get kerberos init command");
StringBuilder kerberosCommandBuilder = new StringBuilder();
boolean hadoopKerberosState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false);
boolean hadoopKerberosState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
if (hadoopKerberosState) {
kerberosCommandBuilder.append("export KRB5_CONFIG=")
.append(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))
.append("\n\n")
.append(String.format("kinit -k -t %s %s || true",PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)))
.append(String.format("kinit -k -t %s %s || true", PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)))
.append("\n\n");
logger.info("kerberos init command: {}", kerberosCommandBuilder);
}
@ -360,7 +130,7 @@ public class ProcessUtils {
File f = new File(commandFile);
if (!f.exists()) {
FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
org.apache.commons.io.FileUtils.writeStringToFile(new File(commandFile), sb.toString(), StandardCharsets.UTF_8);
}
String runCmd = String.format("%s %s", Constants.SH, commandFile);
@ -372,35 +142,6 @@ public class ProcessUtils {
}
}
/**
* kill tasks according to different task types.
*
* @param taskExecutionContext taskExecutionContext
*/
public static void kill(TaskExecutionContext taskExecutionContext) {
try {
int processId = taskExecutionContext.getProcessId();
if (processId == 0) {
logger.error("process kill failed, process id :{}, task id:{}",
processId, taskExecutionContext.getTaskInstanceId());
return;
}
String pidsStr = getPidsStr(processId);
if (StringUtils.isNotEmpty(pidsStr)) {
String cmd = String.format("kill -9 %s", pidsStr);
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", processId, cmd);
OSUtils.exeCmd(cmd);
}
} catch (Exception e) {
logger.error("kill task failed", e);
}
// find log and kill yarn job
killYarnJob(taskExecutionContext);
}
/**
* get pids str.
*
@ -436,6 +177,7 @@ public class ProcessUtils {
/**
* find logs and kill yarn tasks.
*
* @param taskExecutionContext taskExecutionContext
* @return yarn application ids
*/
@ -448,7 +190,7 @@ public class ProcessUtils {
Constants.RPC_PORT,
taskExecutionContext.getLogPath());
}
if (StringUtils.isNotEmpty(log)) {
if (!StringUtils.isEmpty(log)) {
if (StringUtils.isEmpty(taskExecutionContext.getExecutePath())) {
taskExecutionContext.setExecutePath(FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(),
taskExecutionContext.getProcessDefineCode(),

7
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java

@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand;
@ -40,6 +39,8 @@ import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
import org.apache.commons.lang.StringUtils;
import java.util.Collections;
import java.util.List;
@ -122,7 +123,7 @@ public class TaskKillProcessor implements NettyRequestProcessor {
}
String pidsStr = ProcessUtils.getPidsStr(taskExecutionContext.getProcessId());
if (StringUtils.isNotEmpty(pidsStr)) {
if (!StringUtils.isEmpty(pidsStr)) {
String cmd = String.format("kill -9 %s", pidsStr);
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd);
@ -180,7 +181,7 @@ public class TaskKillProcessor implements NettyRequestProcessor {
logger.info("view log host : {},logPath : {}", host, logPath);
String log = logClient.viewLog(host, Constants.RPC_PORT, logPath);
List<String> appIds = Collections.emptyList();
if (StringUtils.isNotEmpty(log)) {
if (!StringUtils.isEmpty(log)) {
appIds = LoggerUtils.getAppIds(log, logger);
if (StringUtils.isEmpty(executePath)) {
logger.error("task instance execute path is empty");

3
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java

@ -26,12 +26,13 @@ import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
import org.apache.dolphinscheduler.server.registry.HeartBeatTask;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.lang.StringUtils;
import java.util.Date;
import java.util.Set;
import java.util.StringJoiner;

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java

@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.RetryerUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand;
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
@ -42,6 +41,7 @@ import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.io.IOException;

14
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java

@ -17,14 +17,16 @@
package org.apache.dolphinscheduler.server.log;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -46,7 +48,7 @@ public class LoggerServerTest {
@Test
public void testRollViewLog() throws IOException {
String expectedTmpDemoString = "testRolloViewLog";
FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset());
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset());
String resultTmpDemoString = this.logClientService.rollViewLog(
"localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000);
@ -59,7 +61,7 @@ public class LoggerServerTest {
@Test
public void testRemoveTaskLog() throws IOException {
String expectedTmpRemoveString = "testRemoveTaskLog";
FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset());
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset());
Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt");

5
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java

@ -17,12 +17,13 @@
package org.apache.dolphinscheduler.server.master.dispatch.host;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -59,7 +60,7 @@ public class RoundRobinHostManagerTest {
Mockito.when(serverNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22"));
ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000);
Host host = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress()));
Assert.assertTrue(!StringUtils.isEmpty(host.getAddress()));
Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22"));
}
}

48
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import java.util.ArrayList;
import java.util.List;
@ -64,53 +63,6 @@ public class ProcessUtilsTest {
Assert.assertEquals("", pidListMac);
}
@Test
public void testBuildCommandStr() {
List<String> commands = new ArrayList<>();
commands.add("sudo");
commands.add("-u");
commands.add("tenantCode");
//allowAmbiguousCommands false
Assert.assertEquals("sudo -u tenantCode", ProcessUtils.buildCommandStr(commands));
//quota
commands.clear();
commands.add("\"sudo\"");
Assert.assertEquals("\"sudo\"", ProcessUtils.buildCommandStr(commands));
//allowAmbiguousCommands true
commands.clear();
commands.add("sudo");
System.setProperty("jdk.lang.Process.allowAmbiguousCommands", "false");
Assert.assertEquals("\"sudo\"", ProcessUtils.buildCommandStr(commands));
}
@Test
public void testKill() {
//get taskExecutionContext
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
//process id eq 0
taskExecutionContext.setProcessId(0);
ProcessUtils.kill(taskExecutionContext);
//process id not eq 0
taskExecutionContext.setProcessId(1);
PowerMockito.mockStatic(OSUtils.class);
try {
when(OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, 1))).thenReturn("1111");
when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, 1))).thenReturn("1111");
when(OSUtils.exeCmd("sudo -u tenantCode kill -9")).thenReturn("1111");
} catch (Exception e) {
e.printStackTrace();
}
taskExecutionContext.setHost("127.0.0.1:8888");
taskExecutionContext.setLogPath("/log/1.log");
taskExecutionContext.setTenantCode("tenantCode");
ProcessUtils.kill(taskExecutionContext);
Assert.assertEquals(1, taskExecutionContext.getProcessId());
}
@Test
public void testGetKerberosInitCommand() {
PowerMockito.mockStatic(PropertyUtils.class);

5
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -59,7 +59,6 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.entity.DagData;
@ -106,6 +105,8 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@ -127,8 +128,6 @@ import org.springframework.transaction.annotation.Transactional;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.facebook.presto.jdbc.internal.guava.collect.Lists;
import com.cronutils.model.Cron;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java

@ -63,13 +63,13 @@ import static org.quartz.TriggerBuilder.newTrigger;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Date;

3
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java

@ -23,7 +23,6 @@ import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHED
import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader;
import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig;
import org.apache.dolphinscheduler.spi.register.Registry;
@ -32,6 +31,8 @@ import org.apache.dolphinscheduler.spi.register.RegistryException;
import org.apache.dolphinscheduler.spi.register.RegistryPluginManager;
import org.apache.dolphinscheduler.spi.register.SubscribeListener;
import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

3
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java

@ -32,7 +32,8 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.ResInfo;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Collections;

58
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java

@ -23,7 +23,6 @@ import static org.apache.dolphinscheduler.spi.task.TaskConstants.SH;
import org.apache.dolphinscheduler.plugin.task.util.LoggerUtils;
import org.apache.dolphinscheduler.plugin.task.util.OSUtils;
import org.apache.dolphinscheduler.plugin.task.util.ThreadUtils;
import org.apache.dolphinscheduler.spi.task.TaskConstants;
import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
@ -41,6 +40,8 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.regex.Matcher;
@ -48,6 +49,8 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* abstract command executor
*/
@ -132,7 +135,6 @@ public abstract class AbstractCommandExecutor {
processBuilder.command(command);
process = processBuilder.start();
// print command
printCommand(command);
}
@ -159,7 +161,7 @@ public abstract class AbstractCommandExecutor {
// parse process output
parseProcessOutput(process);
Integer processId = getProcessId(process);
int processId = getProcessId(process);
result.setProcessId(processId);
@ -242,7 +244,6 @@ public abstract class AbstractCommandExecutor {
*
* @param processId process id
* @return process is alive
* @throws InterruptedException interrupted exception
*/
private boolean softKill(int processId) {
@ -281,20 +282,8 @@ public abstract class AbstractCommandExecutor {
}
}
/**
* print command
*
* @param commands process builder
*/
private void printCommand(List<String> commands) {
String cmdStr;
try {
cmdStr = ProcessUtils.buildCommandStr(commands);
logger.info("task run command:\n{}", cmdStr);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
logger.info("task run command: {}", String.join(" ", commands));
}
/**
@ -320,7 +309,7 @@ public abstract class AbstractCommandExecutor {
*/
private void parseProcessOutput(Process process) {
String threadLoggerInfoName = String.format(LoggerUtils.TASK_LOGGER_THREAD_NAME + "-%s", taskRequest.getTaskAppId());
ExecutorService getOutputLogService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName + "-" + "getOutputLogService");
ExecutorService getOutputLogService = newDaemonSingleThreadExecutor(threadLoggerInfoName + "-" + "getOutputLogService");
getOutputLogService.submit(() -> {
try (BufferedReader inReader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
String line;
@ -343,7 +332,7 @@ public abstract class AbstractCommandExecutor {
getOutputLogService.shutdown();
ExecutorService parseProcessOutputExecutorService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName);
ExecutorService parseProcessOutputExecutorService = newDaemonSingleThreadExecutor(threadLoggerInfoName);
parseProcessOutputExecutorService.submit(() -> {
try {
long lastFlushTime = System.currentTimeMillis();
@ -364,10 +353,6 @@ public abstract class AbstractCommandExecutor {
parseProcessOutputExecutorService.shutdown();
}
public int getProcessId() {
return getProcessId(process);
}
/**
* get app links
*
@ -378,7 +363,7 @@ public abstract class AbstractCommandExecutor {
List<String> logs = convertFile2List(logPath);
List<String> appIds = new ArrayList<>();
/**
/*
* analysis log?get submited yarn application id
*/
for (String log : logs) {
@ -398,7 +383,7 @@ public abstract class AbstractCommandExecutor {
* @return line list
*/
private List<String> convertFile2List(String filename) {
List lineList = new ArrayList<String>(100);
List<String> lineList = new ArrayList<>(100);
File file = new File(filename);
if (!file.exists()) {
@ -477,12 +462,11 @@ public abstract class AbstractCommandExecutor {
private long flush(long lastFlushTime) {
long now = System.currentTimeMillis();
/**
/*
* when log buffer siz or flush time reach condition , then flush
*/
if (logBuffer.size() >= TaskConstants.DEFAULT_LOG_ROWS_NUM || now - lastFlushTime > TaskConstants.DEFAULT_LOG_FLUSH_INTERVAL) {
lastFlushTime = now;
/** log handle */
logHandler.accept(logBuffer);
logBuffer.clear();
@ -490,21 +474,15 @@ public abstract class AbstractCommandExecutor {
return lastFlushTime;
}
protected List<String> commandOptions() {
return Collections.emptyList();
}
protected abstract String buildCommandFilePath();
protected abstract String commandInterpreter();
protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException;
public String getTaskResultString() {
return taskResultString;
}
public void setTaskResultString(String taskResultString) {
this.taskResultString = taskResultString;
ExecutorService newDaemonSingleThreadExecutor(String threadName) {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat(threadName)
.build();
return Executors.newSingleThreadExecutor(threadFactory);
}
}
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save