Browse Source

Merge branch 'dev' into dev-FTS

pull/3/MERGE
Zhou Zheng 4 years ago
parent
commit
cf9576b74a
  1. 2
      CONTRIBUTING.md
  2. 2
      docker/build/conf/zookeeper/zoo.cfg
  3. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java
  4. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
  5. 6
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  6. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
  7. 21
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java
  8. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  9. 5
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
  10. 2
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java
  11. 37
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  12. 23
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
  13. 3
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
  14. 5
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  15. 3
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  16. 32
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
  17. 205
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
  18. 43
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java
  19. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
  20. 41
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java
  21. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java
  22. 7
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
  23. 96
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
  24. 34
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  25. 224
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java
  26. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java
  27. 72
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
  28. 62
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java
  29. 32
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
  30. 25
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
  31. 818
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  32. 406
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
  33. 19
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java
  34. 48
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
  35. 14
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java
  36. 222
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
  37. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
  38. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java
  39. 12
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  40. 100
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java
  41. 22
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java
  42. 12
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml
  43. 4
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java
  44. 36
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java
  45. 23
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java
  46. 9
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java
  47. 20
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java
  48. 12
      dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java
  49. 2
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java
  50. 4
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java
  51. 20
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java
  52. 62
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
  53. 76
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java
  54. 30
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
  55. 159
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
  56. 46
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java
  57. 7
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java
  58. 66
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java
  59. 3
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue
  60. 2
      dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue
  61. 85
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue
  62. 94
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/relatedItems.vue
  63. 59
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue
  64. 32
      dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js
  65. 6
      dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js
  66. 6
      dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js
  67. 4
      install.sh
  68. 3
      pom.xml
  69. 19
      script/dolphinscheduler-daemon.sh
  70. 4
      script/start-all.sh
  71. 59
      script/status-all.sh
  72. 4
      script/stop-all.sh

2
CONTRIBUTING.md

@ -56,7 +56,7 @@ If remote branch has a new branch `DEV-1.0`, you need to synchronize this branch
``` ```
git checkout -b dev-1.0 upstream/dev-1.0 git checkout -b dev-1.0 upstream/dev-1.0
git push --set-upstream origin dev1.0 git push --set-upstream origin dev-1.0
``` ```
## Create your feature branch ## Create your feature branch

2
docker/build/conf/zookeeper/zoo.cfg

@ -43,3 +43,5 @@ clientPort=2181
# Purge task interval in hours # Purge task interval in hours
# Set to "0" to disable auto purge feature # Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1 #autopurge.purgeInterval=1
#Four Letter Words commands:stat,ruok,conf,isro
4lw.commands.whitelist=*

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java

@ -220,11 +220,7 @@ public class ServiceModelToSwagger2MapperImpl extends ServiceModelToSwagger2Mapp
if (resourceListing == null) { if (resourceListing == null) {
return null; return null;
} }
ApiInfo info = resourceListing.getInfo(); return resourceListing.getInfo();
if (info == null) {
return null;
}
return info;
} }
protected List<Tag> tagSetToTagList(Set<springfox.documentation.service.Tag> set) { protected List<Tag> tagSetToTagList(Set<springfox.documentation.service.Tag> set) {

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java

@ -57,11 +57,10 @@ public class ResourceFilter implements IFilter {
* @return file filtered by suffix * @return file filtered by suffix
*/ */
public Set<Resource> fileFilter(){ public Set<Resource> fileFilter(){
Set<Resource> resources = resourceList.stream().filter(t -> { return resourceList.stream().filter(t -> {
String alias = t.getAlias(); String alias = t.getAlias();
return alias.endsWith(suffix); return alias.endsWith(suffix);
}).collect(Collectors.toSet()); }).collect(Collectors.toSet());
return resources;
} }
/** /**

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -193,10 +193,10 @@ public enum Status {
BATCH_COPY_PROCESS_DEFINITION_ERROR(10159, "batch copy process definition error", "复制工作流错误"), BATCH_COPY_PROCESS_DEFINITION_ERROR(10159, "batch copy process definition error", "复制工作流错误"),
BATCH_MOVE_PROCESS_DEFINITION_ERROR(10160, "batch move process definition error", "移动工作流错误"), BATCH_MOVE_PROCESS_DEFINITION_ERROR(10160, "batch move process definition error", "移动工作流错误"),
QUERY_WORKFLOW_LINEAGE_ERROR(10161, "query workflow lineage error", "查询血缘失败"), QUERY_WORKFLOW_LINEAGE_ERROR(10161, "query workflow lineage error", "查询血缘失败"),
FORCE_TASK_SUCCESS_ERROR(10162, "force task success error", "强制成功任务实例错误"),
TASK_INSTANCE_STATE_OPERATION_ERROR(10163, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}],无法执行强制成功操作"),
DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10162,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败,有[{0}]个运行中的工作流实例正在使用"),
FORCE_TASK_SUCCESS_ERROR(10163, "force task success error", "强制成功任务实例错误"),
TASK_INSTANCE_STATE_OPERATION_ERROR(10164, "the status of task instance {0} is {1},Cannot perform force success operation", "任务实例[{0}]的状态是[{1}],无法执行强制成功操作"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java

@ -189,10 +189,7 @@ public class DataSourceService extends BaseService {
private boolean checkName(String name) { private boolean checkName(String name) {
List<DataSource> queryDataSource = dataSourceMapper.queryDataSourceByName(name.trim()); List<DataSource> queryDataSource = dataSourceMapper.queryDataSourceByName(name.trim());
if (queryDataSource != null && queryDataSource.size() > 0) { return queryDataSource != null && queryDataSource.size() > 0;
return true;
}
return false;
} }

21
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java

@ -24,7 +24,6 @@ import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE;
import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; import static org.apache.dolphinscheduler.common.Constants.TASK_LIST;
import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.GanttDto;
import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.dto.gantt.Task;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
@ -711,4 +710,24 @@ public class ProcessInstanceService extends BaseService {
return DagHelper.buildDagGraph(processDag); return DagHelper.buildDagGraph(processDag);
} }
/**
* query process instance by processDefinitionId and stateArray
* @param processDefinitionId processDefinitionId
* @param states states array
* @return process instance list
*/
public List<ProcessInstance> queryByProcessDefineIdAndStatus(int processDefinitionId, int[] states) {
return processInstanceMapper.queryByProcessDefineIdAndStatus(processDefinitionId, states);
}
/**
* query process instance by processDefinitionId
* @param processDefinitionId processDefinitionId
* @param size size
* @return process instance list
*/
public List<ProcessInstance> queryByProcessDefineId(int processDefinitionId,int size) {
return processInstanceMapper.queryByProcessDefineId(processDefinitionId, size);
}
} }

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -272,10 +272,7 @@ public class ResourcesService extends BaseService {
private boolean checkResourceExists(String fullName, int userId, int type ){ private boolean checkResourceExists(String fullName, int userId, int type ){
List<Resource> resources = resourcesMapper.queryResourceList(fullName, userId, type); List<Resource> resources = resourcesMapper.queryResourceList(fullName, userId, type);
if (resources != null && resources.size() > 0) { return resources != null && resources.size() > 0;
return true;
}
return false;
} }

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java

@ -136,10 +136,7 @@ public class UdfFuncService extends BaseService{
*/ */
private boolean checkUdfFuncNameExists(String name){ private boolean checkUdfFuncNameExists(String name){
List<UdfFunc> resource = udfFuncMapper.queryUdfByIdStr(null, name); List<UdfFunc> resource = udfFuncMapper.queryUdfByIdStr(null, name);
if(resource != null && resource.size() > 0){ return resource != null && resource.size() > 0;
return true;
}
return false;
} }

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java

@ -120,7 +120,7 @@ public class AccessTokenServiceImpl extends BaseService implements AccessTokenSe
*/ */
public Map<String, Object> generateToken(int userId, String expireTime) { public Map<String, Object> generateToken(int userId, String expireTime) {
Map<String, Object> result = new HashMap<>(5); Map<String, Object> result = new HashMap<>(5);
String token = EncryptionUtils.getMd5(userId + expireTime + String.valueOf(System.currentTimeMillis())); String token = EncryptionUtils.getMd5(userId + expireTime + System.currentTimeMillis());
result.put(Constants.DATA_LIST, token); result.put(Constants.DATA_LIST, token);
putMsg(result, Status.SUCCESS); putMsg(result, Status.SUCCESS);
return result; return result;

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.BaseService; import org.apache.dolphinscheduler.api.service.BaseService;
import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService;
import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService; import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService;
import org.apache.dolphinscheduler.api.service.ProcessInstanceService;
import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.service.ProjectService;
import org.apache.dolphinscheduler.api.service.SchedulerService; import org.apache.dolphinscheduler.api.service.SchedulerService;
import org.apache.dolphinscheduler.api.utils.CheckUtils; import org.apache.dolphinscheduler.api.utils.CheckUtils;
@ -64,7 +65,6 @@ import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
@ -134,7 +134,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
private ProcessDefinitionMapper processDefineMapper; private ProcessDefinitionMapper processDefineMapper;
@Autowired @Autowired
private ProcessInstanceMapper processInstanceMapper; private ProcessInstanceService processInstanceService;
@Autowired @Autowired
private TaskInstanceMapper taskInstanceMapper; private TaskInstanceMapper taskInstanceMapper;
@ -488,6 +488,12 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
putMsg(result, Status.PROCESS_DEFINE_STATE_ONLINE, processDefinitionId); putMsg(result, Status.PROCESS_DEFINE_STATE_ONLINE, processDefinitionId);
return result; return result;
} }
// check process instances is already running
List<ProcessInstance> processInstances = processInstanceService.queryByProcessDefineIdAndStatus(processDefinitionId, Constants.NOT_TERMINATED_STATES);
if (CollectionUtils.isNotEmpty(processInstances)) {
putMsg(result, Status.DELETE_PROCESS_DEFINITION_BY_ID_FAIL,processInstances.size());
return result;
}
// get the timing according to the process definition // get the timing according to the process definition
List<Schedule> schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); List<Schedule> schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId);
@ -1247,7 +1253,7 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
/** /**
* List of process instances * List of process instances
*/ */
List<ProcessInstance> processInstanceList = processInstanceMapper.queryByProcessDefineId(processId, limit); List<ProcessInstance> processInstanceList = processInstanceService.queryByProcessDefineId(processId, limit);
for (ProcessInstance processInstance : processInstanceList) { for (ProcessInstance processInstance : processInstanceList) {
processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime()));
@ -1321,14 +1327,11 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (CollectionUtils.isNotEmpty(postNodeList)) { if (CollectionUtils.isNotEmpty(postNodeList)) {
for (String nextNodeName : postNodeList) { for (String nextNodeName : postNodeList) {
List<TreeViewDto> treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); List<TreeViewDto> treeViewDtoList = waitingRunningNodeMap.get(nextNodeName);
if (CollectionUtils.isNotEmpty(treeViewDtoList)) { if (CollectionUtils.isEmpty(treeViewDtoList)) {
treeViewDtoList.add(treeViewDto);
waitingRunningNodeMap.put(nextNodeName, treeViewDtoList);
} else {
treeViewDtoList = new ArrayList<>(); treeViewDtoList = new ArrayList<>();
treeViewDtoList.add(treeViewDto);
waitingRunningNodeMap.put(nextNodeName, treeViewDtoList);
} }
treeViewDtoList.add(treeViewDto);
waitingRunningNodeMap.put(nextNodeName, treeViewDtoList);
} }
} }
runningNodeMap.remove(nodeName); runningNodeMap.remove(nodeName);
@ -1690,16 +1693,16 @@ public class ProcessDefinitionServiceImpl extends BaseService implements
if (processDefinition == null) { if (processDefinition == null) {
putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId);
return result; return result;
}
processDefinition.setProjectId(targetProject.getId());
processDefinition.setUpdateTime(new Date());
if (processDefineMapper.updateById(processDefinition) > 0) {
putMsg(result, Status.SUCCESS);
} else { } else {
processDefinition.setProjectId(targetProject.getId()); putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR);
processDefinition.setUpdateTime(new Date());
if (processDefineMapper.updateById(processDefinition) > 0) {
putMsg(result, Status.SUCCESS);
} else {
putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR);
}
return result;
} }
return result;
} }
/** /**

23
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java

@ -16,7 +16,6 @@
*/ */
package org.apache.dolphinscheduler.api.controller; package org.apache.dolphinscheduler.api.controller;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
@ -53,8 +52,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -77,8 +74,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -280,8 +275,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -302,8 +295,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -323,8 +314,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -343,8 +332,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -364,8 +351,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -385,8 +370,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -405,8 +388,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -426,8 +407,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
@ -445,8 +424,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{
.andReturn(); .andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java

@ -49,10 +49,9 @@ public class ResourceFilterTest {
allList.add(resource6); allList.add(resource6);
allList.add(resource7); allList.add(resource7);
ResourceFilter resourceFilter = new ResourceFilter(".jar",allList); ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
List<Resource> resourceList = resourceFilter.filter(); List<Resource> resourceList = resourceFilter.filter();
Assert.assertNotNull(resourceList); Assert.assertNotNull(resourceList);
resourceList.stream().forEach(t-> logger.info(t.toString())); resourceList.forEach(t -> logger.info(t.toString()));
} }
} }

5
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -41,7 +41,6 @@ import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
@ -94,7 +93,7 @@ public class ProcessDefinitionServiceTest {
private ProcessService processService; private ProcessService processService;
@Mock @Mock
private ProcessInstanceMapper processInstanceMapper; private ProcessInstanceService processInstanceService;
@Mock @Mock
private TaskInstanceMapper taskInstanceMapper; private TaskInstanceMapper taskInstanceMapper;
@ -734,7 +733,7 @@ public class ProcessDefinitionServiceTest {
//task instance not exist //task instance not exist
Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition);
Mockito.when(processInstanceMapper.queryByProcessDefineId(46, 10)).thenReturn(processInstanceList); Mockito.when(processInstanceService.queryByProcessDefineId(46, 10)).thenReturn(processInstanceList);
Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(null); Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(null);
Map<String, Object> taskNullRes = processDefinitionService.viewTree(46, 10); Map<String, Object> taskNullRes = processDefinitionService.viewTree(46, 10);
Assert.assertEquals(Status.SUCCESS, taskNullRes.get(Constants.STATUS)); Assert.assertEquals(Status.SUCCESS, taskNullRes.get(Constants.STATUS));

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common; package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -27,7 +28,7 @@ import java.util.regex.Pattern;
public final class Constants { public final class Constants {
private Constants() { private Constants() {
throw new IllegalStateException("Constants class"); throw new UnsupportedOperationException("Construct Constants");
} }
/** /**

32
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java

@ -14,13 +14,19 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap; import org.apache.commons.beanutils.BeanMap;
import org.apache.commons.lang.StringUtils;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/** /**
* Provides utility methods and decorators for {@link Collection} instances. * Provides utility methods and decorators for {@link Collection} instances.
@ -37,8 +43,9 @@ import java.util.*;
public class CollectionUtils { public class CollectionUtils {
private CollectionUtils() { private CollectionUtils() {
throw new IllegalStateException("CollectionUtils class"); throw new UnsupportedOperationException("Construct CollectionUtils");
} }
/** /**
* Returns a new {@link Collection} containing <i>a</i> minus a subset of * Returns a new {@link Collection} containing <i>a</i> minus a subset of
* <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate * <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate
@ -71,7 +78,7 @@ public class CollectionUtils {
/** /**
* String to map * String to map
* *
* @param str string * @param str string
* @param separator separator * @param separator separator
* @return string to map * @return string to map
*/ */
@ -82,7 +89,7 @@ public class CollectionUtils {
/** /**
* String to map * String to map
* *
* @param str string * @param str string
* @param separator separator * @param separator separator
* @param keyPrefix prefix * @param keyPrefix prefix
* @return string to map * @return string to map
@ -112,7 +119,6 @@ public class CollectionUtils {
return map; return map;
} }
/** /**
* Helper class to easily access cardinality properties of two collections. * Helper class to easily access cardinality properties of two collections.
* *
@ -137,8 +143,8 @@ public class CollectionUtils {
* @param b the second collection * @param b the second collection
*/ */
public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) { public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) {
cardinalityA = CollectionUtils.<O>getCardinalityMap(a); cardinalityA = CollectionUtils.getCardinalityMap(a);
cardinalityB = CollectionUtils.<O>getCardinalityMap(b); cardinalityB = CollectionUtils.getCardinalityMap(b);
} }
/** /**
@ -227,7 +233,7 @@ public class CollectionUtils {
* Only those elements present in the collection will appear as * Only those elements present in the collection will appear as
* keys in the map. * keys in the map.
* *
* @param <O> the type of object in the returned {@link Map}. This is a super type of O * @param <O> the type of object in the returned {@link Map}. This is a super type of O
* @param coll the collection to get the cardinality map for, must not be null * @param coll the collection to get the cardinality map for, must not be null
* @return the populated cardinality map * @return the populated cardinality map
*/ */
@ -239,9 +245,9 @@ public class CollectionUtils {
return count; return count;
} }
/** /**
* Removes certain attributes of each object in the list * Removes certain attributes of each object in the list
*
* @param originList origin list * @param originList origin list
* @param exclusionSet exclusion set * @param exclusionSet exclusion set
* @param <T> T * @param <T> T
@ -258,8 +264,8 @@ public class CollectionUtils {
Map<String, Object> instanceMap; Map<String, Object> instanceMap;
for (T instance : originList) { for (T instance : originList) {
Map<String, Object> dataMap = new BeanMap(instance); Map<String, Object> dataMap = new BeanMap(instance);
instanceMap = new LinkedHashMap<>(16,0.75f,true); instanceMap = new LinkedHashMap<>(16, 0.75f, true);
for (Map.Entry<String, Object> entry: dataMap.entrySet()) { for (Map.Entry<String, Object> entry : dataMap.entrySet()) {
if (exclusionSet.contains(entry.getKey())) { if (exclusionSet.contains(entry.getKey())) {
continue; continue;
} }

205
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java

@ -14,124 +14,129 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* common utils * common utils
*/ */
public class CommonUtils { public class CommonUtils {
private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class);
private static final Base64 BASE64 = new Base64(); private static final Base64 BASE64 = new Base64();
private CommonUtils() { private CommonUtils() {
throw new IllegalStateException("CommonUtils class"); throw new UnsupportedOperationException("Construct CommonUtils");
}
/**
* @return get the path of system environment variables
*/
public static String getSystemEnvPath() {
String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
if (StringUtils.isEmpty(envPath)) {
URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
if (envDefaultPath != null){
envPath = envDefaultPath.getPath();
logger.debug("env path :{}", envPath);
}else{
envPath = "/etc/profile";
}
} }
return envPath; /**
} * @return get the path of system environment variables
*/
/** public static String getSystemEnvPath() {
* String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH);
* @return is develop mode if (StringUtils.isEmpty(envPath)) {
*/ URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH);
public static boolean isDevelopMode() {
return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true); if (envDefaultPath != null) {
} envPath = envDefaultPath.getPath();
logger.debug("env path :{}", envPath);
} else {
envPath = "/etc/profile";
/** }
* if upload resource is HDFS and kerberos startup is true , else false }
* @return true if upload resource is HDFS and kerberos startup
*/ return envPath;
public static boolean getKerberosStartupState(){
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
}
/**
* load kerberos configuration
* @throws Exception errors
*/
public static void loadKerberosConf()throws Exception{
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration();
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
} }
}
/**
/** * @return is develop mode
* encode password */
* @param password public static boolean isDevelopMode() {
* @return return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
*/
public static String encodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ;
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
}
/**
* decode password
* @param password
* @return
*/
public static String decodePassword(String password) {
if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; }
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false);
if ( !encryptionEnable){ return password; }
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ;
if(!passwordWithSalt.startsWith(salt)){
logger.warn("There is a password and salt mismatch: {} ",password);
return password;
} }
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ;
}
/**
* if upload resource is HDFS and kerberos startup is true , else false
*
* @return true if upload resource is HDFS and kerberos startup
*/
public static boolean getKerberosStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false);
return resUploadType == ResUploadType.HDFS && kerberosStartupState;
}
/**
* load kerberos configuration
*
* @throws Exception errors
*/
public static void loadKerberosConf() throws Exception {
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
Configuration configuration = new Configuration();
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
}
/**
* encode password
*/
public static String encodePassword(String password) {
if (StringUtils.isEmpty(password)) {
return StringUtils.EMPTY;
}
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
if (!encryptionEnable) {
return password;
}
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8)));
return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8)));
}
/**
* decode password
*/
public static String decodePassword(String password) {
if (StringUtils.isEmpty(password)) {
return StringUtils.EMPTY;
}
//if encryption is not turned on, return directly
boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false);
if (!encryptionEnable) {
return password;
}
// Using Base64 + salt to process password
String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT);
String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8);
if (!passwordWithSalt.startsWith(salt)) {
logger.warn("There is a password and salt mismatch: {} ", password);
return password;
}
return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8);
}
} }

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java

@ -14,37 +14,40 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.util.Arrays; import java.util.Arrays;
import java.util.Objects; import java.util.Objects;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class ConnectionUtils { public class ConnectionUtils {
public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class); public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class);
private ConnectionUtils() { private ConnectionUtils() {
throw new IllegalStateException("ConnectionUtils class"); throw new UnsupportedOperationException("Construct ConnectionUtils");
} }
/** /**
* release resource * release resource
* @param resources resources *
*/ * @param resources resources
public static void releaseResource(AutoCloseable... resources) { */
public static void releaseResource(AutoCloseable... resources) {
if (resources == null || resources.length == 0) { if (resources == null || resources.length == 0) {
return; return;
}
Arrays.stream(resources).filter(Objects::nonNull)
.forEach(resource -> {
try {
resource.close();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});
} }
Arrays.stream(resources).filter(Objects::nonNull)
.forEach(resource -> {
try {
resource.close();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});
}
} }

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java

@ -36,6 +36,10 @@ public class DateUtils {
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils");
}
/** /**
* date to local datetime * date to local datetime
* *

41
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java

@ -20,8 +20,6 @@ import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.enums.DependentRelation;
import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils; import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
@ -29,32 +27,35 @@ import java.util.List;
public class DependentUtils { public class DependentUtils {
private static final Logger logger = LoggerFactory.getLogger(DependentUtils.class); private DependentUtils() {
throw new UnsupportedOperationException("Construct DependentUtils");
}
public static DependResult getDependResultForRelation(DependentRelation relation, public static DependResult getDependResultForRelation(DependentRelation relation,
List<DependResult> dependResultList){ List<DependResult> dependResultList) {
DependResult dependResult = DependResult.SUCCESS; DependResult dependResult = DependResult.SUCCESS;
switch (relation){ switch (relation) {
case AND: case AND:
if(dependResultList.contains(DependResult.FAILED)){ if (dependResultList.contains(DependResult.FAILED)) {
dependResult = DependResult.FAILED; dependResult = DependResult.FAILED;
} if(dependResultList.contains(DependResult.WAITING)){ }
if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING; dependResult = DependResult.WAITING;
} }
break; break;
case OR: case OR:
if(dependResultList.contains(DependResult.SUCCESS)){ if (dependResultList.contains(DependResult.SUCCESS)) {
dependResult = DependResult.SUCCESS; dependResult = DependResult.SUCCESS;
}else if(dependResultList.contains(DependResult.WAITING)){ } else if (dependResultList.contains(DependResult.WAITING)) {
dependResult = DependResult.WAITING; dependResult = DependResult.WAITING;
}else{ } else {
dependResult = DependResult.FAILED; dependResult = DependResult.FAILED;
} }
break; break;
default: default:
break; break;
} }
return dependResult; return dependResult;
} }
@ -62,36 +63,37 @@ public class DependentUtils {
/** /**
* get date interval list by business date and date value. * get date interval list by business date and date value.
*
* @param businessDate business date * @param businessDate business date
* @param dateValue date value * @param dateValue date value
* @return date interval list by business date and date value. * @return date interval list by business date and date value.
*/ */
public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue){ public static List<DateInterval> getDateIntervalList(Date businessDate, String dateValue) {
List<DateInterval> result = new ArrayList<>(); List<DateInterval> result = new ArrayList<>();
switch (dateValue){ switch (dateValue) {
case "currentHour": case "currentHour":
result = DependentDateUtils.getLastHoursInterval(businessDate, 0); result = DependentDateUtils.getLastHoursInterval(businessDate, 0);
break; break;
case "last1Hour": case "last1Hour":
result = DependentDateUtils.getLastHoursInterval(businessDate, 1); result = DependentDateUtils.getLastHoursInterval(businessDate, 1);
break; break;
case "last2Hours": case "last2Hours":
result = DependentDateUtils.getLastHoursInterval(businessDate, 2); result = DependentDateUtils.getLastHoursInterval(businessDate, 2);
break; break;
case "last3Hours": case "last3Hours":
result = DependentDateUtils.getLastHoursInterval(businessDate, 3); result = DependentDateUtils.getLastHoursInterval(businessDate, 3);
break; break;
case "last24Hours": case "last24Hours":
result = DependentDateUtils.getSpecialLastDayInterval(businessDate); result = DependentDateUtils.getSpecialLastDayInterval(businessDate);
break; break;
case "today": case "today":
result = DependentDateUtils.getTodayInterval(businessDate); result = DependentDateUtils.getTodayInterval(businessDate);
break; break;
case "last1Days": case "last1Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 1); result = DependentDateUtils.getLastDayInterval(businessDate, 1);
break; break;
case "last2Days": case "last2Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 2); result = DependentDateUtils.getLastDayInterval(businessDate, 2);
break; break;
case "last3Days": case "last3Days":
result = DependentDateUtils.getLastDayInterval(businessDate, 3); result = DependentDateUtils.getLastDayInterval(businessDate, 3);
@ -144,5 +146,4 @@ public class DependentUtils {
return result; return result;
} }
} }

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
@ -23,14 +24,16 @@ import org.apache.commons.codec.digest.DigestUtils;
*/ */
public class EncryptionUtils { public class EncryptionUtils {
private EncryptionUtils() {
throw new UnsupportedOperationException("Construct EncryptionUtils");
}
/** /**
*
* @param rawStr raw string * @param rawStr raw string
* @return md5(rawStr) * @return md5(rawStr)
*/ */
public static String getMd5(String rawStr) { public static String getMd5(String rawStr) {
return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr); return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr);
} }
} }

7
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java

@ -14,12 +14,15 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
public class EnumUtils { public class EnumUtils {
private EnumUtils() {
throw new UnsupportedOperationException("Construct EnumUtils");
}
public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) { public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) {
if (enumName == null) { if (enumName == null) {
return null; return null;

96
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH; import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
@ -21,6 +22,9 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS
import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE; import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.BufferedWriter; import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@ -36,9 +40,6 @@ import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException; import java.nio.charset.UnsupportedCharsetException;
import java.util.Optional; import java.util.Optional;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -46,12 +47,17 @@ import org.slf4j.LoggerFactory;
* file utils * file utils
*/ */
public class FileUtils { public class FileUtils {
public static final Logger logger = LoggerFactory.getLogger(FileUtils.class); public static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler"); public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH, "/tmp/dolphinscheduler");
public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>(); public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
private FileUtils() {
throw new UnsupportedOperationException("Construct FileUtils");
}
/** /**
* get file suffix * get file suffix
* *
@ -80,7 +86,7 @@ public class FileUtils {
String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename);
File file = new File(fileName); File file = new File(fileName);
if (!file.getParentFile().exists()){ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
@ -97,7 +103,7 @@ public class FileUtils {
public static String getUploadFilename(String tenantCode, String filename) { public static String getUploadFilename(String tenantCode, String filename) {
String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename); String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename);
File file = new File(fileName); File file = new File(fileName);
if (!file.getParentFile().exists()){ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
@ -106,6 +112,7 @@ public class FileUtils {
/** /**
* directory of process execution * directory of process execution
*
* @param projectId project id * @param projectId project id
* @param processDefineId process definition id * @param processDefineId process definition id
* @param processInstanceId process instance id * @param processInstanceId process instance id
@ -114,9 +121,9 @@ public class FileUtils {
*/ */
public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) { public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) {
String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId), String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId),
Integer.toString(processDefineId), Integer.toString(processInstanceId),Integer.toString(taskInstanceId)); Integer.toString(processDefineId), Integer.toString(processInstanceId), Integer.toString(taskInstanceId));
File file = new File(fileName); File file = new File(fileName);
if (!file.getParentFile().exists()){ if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
} }
@ -125,6 +132,7 @@ public class FileUtils {
/** /**
* directory of process instances * directory of process instances
*
* @param projectId project id * @param projectId project id
* @param processDefineId process definition id * @param processDefineId process definition id
* @param processInstanceId process instance id * @param processInstanceId process instance id
@ -150,6 +158,7 @@ public class FileUtils {
/** /**
* create directory and user * create directory and user
*
* @param execLocalPath execute local path * @param execLocalPath execute local path
* @param userName user name * @param userName user name
* @throws IOException errors * @throws IOException errors
@ -190,12 +199,11 @@ public class FileUtils {
OSUtils.taskLoggerThreadLocal.remove(); OSUtils.taskLoggerThreadLocal.remove();
} }
/** /**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir * write content to file ,if parent path not exists, it will do one's utmost to mkdir
* *
* @param content content * @param content content
* @param filePath target file path * @param filePath target file path
* @return true if write success * @return true if write success
*/ */
public static boolean writeContent2File(String content, String filePath) { public static boolean writeContent2File(String content, String filePath) {
@ -231,13 +239,13 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist. * Writes a String to a file creating the file if it does not exist.
* * <p>
* NOTE: As from v1.3, the parent directories of the file will be created * NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist. * if they do not exist.
* *
* @param file the file to write * @param file the file to write
* @param data the content to write to the file * @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default * @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
* @since 2.4 * @since 2.4
@ -248,13 +256,13 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist. * Writes a String to a file creating the file if it does not exist.
* * <p>
* NOTE: As from v1.3, the parent directories of the file will be created * NOTE: As from v1.3, the parent directories of the file will be created
* if they do not exist. * if they do not exist.
* *
* @param file the file to write * @param file the file to write
* @param data the content to write to the file * @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default * @param encoding the encoding to use, {@code null} means platform default
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
* @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
*/ */
@ -265,9 +273,9 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist. * Writes a String to a file creating the file if it does not exist.
* *
* @param file the file to write * @param file the file to write
* @param data the content to write to the file * @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default * @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the * @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting * end of the file rather than overwriting
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
@ -287,15 +295,14 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist. * Writes a String to a file creating the file if it does not exist.
* *
* @param file the file to write * @param file the file to write
* @param data the content to write to the file * @param data the content to write to the file
* @param encoding the encoding to use, {@code null} means platform default * @param encoding the encoding to use, {@code null} means platform default
* @param append if {@code true}, then the String will be added to the * @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting * end of the file rather than overwriting
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
* @throws UnsupportedCharsetException * @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not
* thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not * supported by the VM
* supported by the VM
* @since 2.1 * @since 2.1
*/ */
public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException { public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException {
@ -305,8 +312,8 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM. * Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
* *
* @param file the file to write * @param file the file to write
* @param data the content to write to the file * @param data the content to write to the file
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
*/ */
public static void writeStringToFile(File file, String data) throws IOException { public static void writeStringToFile(File file, String data) throws IOException {
@ -316,8 +323,8 @@ public class FileUtils {
/** /**
* Writes a String to a file creating the file if it does not exist using the default encoding for the VM. * Writes a String to a file creating the file if it does not exist using the default encoding for the VM.
* *
* @param file the file to write * @param file the file to write
* @param data the content to write to the file * @param data the content to write to the file
* @param append if {@code true}, then the String will be added to the * @param append if {@code true}, then the String will be added to the
* end of the file rather than overwriting * end of the file rather than overwriting
* @throws IOException in case of an I/O error * @throws IOException in case of an I/O error
@ -340,7 +347,7 @@ public class FileUtils {
* An exception is thrown if the file exists but cannot be written to. * An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created. * An exception is thrown if the parent directory cannot be created.
* *
* @param file the file to open for output, must not be {@code null} * @param file the file to open for output, must not be {@code null}
* @return a new {@link FileOutputStream} for the specified file * @return a new {@link FileOutputStream} for the specified file
* @throws IOException if the file object is a directory * @throws IOException if the file object is a directory
* @throws IOException if the file cannot be written to * @throws IOException if the file cannot be written to
@ -364,7 +371,7 @@ public class FileUtils {
* An exception is thrown if the file exists but cannot be written to. * An exception is thrown if the file exists but cannot be written to.
* An exception is thrown if the parent directory cannot be created. * An exception is thrown if the parent directory cannot be created.
* *
* @param file the file to open for output, must not be {@code null} * @param file the file to open for output, must not be {@code null}
* @param append if {@code true}, then bytes will be added to the * @param append if {@code true}, then bytes will be added to the
* end of the file rather than overwriting * end of the file rather than overwriting
* @return a new {@link FileOutputStream} for the specified file * @return a new {@link FileOutputStream} for the specified file
@ -384,15 +391,15 @@ public class FileUtils {
} else { } else {
File parent = file.getParentFile(); File parent = file.getParentFile();
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
throw new IOException("Directory '" + parent + "' could not be created"); throw new IOException("Directory '" + parent + "' could not be created");
} }
} }
return new FileOutputStream(file, append); return new FileOutputStream(file, append);
} }
/** /**
* deletes a directory recursively * deletes a directory recursively
*
* @param dir directory * @param dir directory
* @throws IOException in case deletion is unsuccessful * @throws IOException in case deletion is unsuccessful
*/ */
@ -420,17 +427,18 @@ public class FileUtils {
/** /**
* Gets all the parent subdirectories of the parentDir directory * Gets all the parent subdirectories of the parentDir directory
*
* @param parentDir parent dir * @param parentDir parent dir
* @return all dirs * @return all dirs
*/ */
public static File[] getAllDir(String parentDir){ public static File[] getAllDir(String parentDir) {
if(parentDir == null || "".equals(parentDir)) { if (parentDir == null || "".equals(parentDir)) {
throw new RuntimeException("parentDir can not be empty"); throw new RuntimeException("parentDir can not be empty");
} }
File file = new File(parentDir); File file = new File(parentDir);
if(!file.exists() || !file.isDirectory()) { if (!file.exists() || !file.isDirectory()) {
throw new RuntimeException("parentDir not exist, or is not a directory:"+parentDir); throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
} }
return file.listFiles(File::isDirectory); return file.listFiles(File::isDirectory);
@ -438,6 +446,7 @@ public class FileUtils {
/** /**
* Get Content * Get Content
*
* @param inputStream input stream * @param inputStream input stream
* @return string of input stream * @return string of input stream
*/ */
@ -447,15 +456,14 @@ public class FileUtils {
ByteArrayOutputStream output = new ByteArrayOutputStream(); ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[1024]; byte[] buffer = new byte[1024];
int length; int length;
while ((length= inputStream.read(buffer)) != -1) { while ((length = inputStream.read(buffer)) != -1) {
output.write(buffer,0,length); output.write(buffer, 0, length);
} }
return output.toString(); return output.toString();
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(),e); logger.error(e.getMessage(), e);
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
} }

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -195,7 +195,7 @@ public class HadoopUtils implements Closeable {
*/ */
String appUrl = ""; String appUrl = "";
if (StringUtils.isEmpty(rmHaIds)){ if (StringUtils.isEmpty(rmHaIds)) {
//single resourcemanager enabled //single resourcemanager enabled
appUrl = appAddress; appUrl = appAddress;
yarnEnabled = true; yarnEnabled = true;
@ -206,7 +206,7 @@ public class HadoopUtils implements Closeable {
logger.info("application url : {}", appUrl); logger.info("application url : {}", appUrl);
} }
if(StringUtils.isBlank(appUrl)){ if (StringUtils.isBlank(appUrl)) {
throw new Exception("application url is blank"); throw new Exception("application url is blank");
} }
return String.format(appUrl, applicationId); return String.format(appUrl, applicationId);
@ -417,25 +417,33 @@ public class HadoopUtils implements Closeable {
String applicationUrl = getApplicationUrl(applicationId); String applicationUrl = getApplicationUrl(applicationId);
logger.info("applicationUrl={}", applicationUrl); logger.info("applicationUrl={}", applicationUrl);
String responseContent ; String responseContent;
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) { if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) {
responseContent = KerberosHttpClient.get(applicationUrl); responseContent = KerberosHttpClient.get(applicationUrl);
} else { } else {
responseContent = HttpUtils.get(applicationUrl); responseContent = HttpUtils.get(applicationUrl);
} }
if (responseContent != null) { if (responseContent != null) {
ObjectNode jsonObject = JSONUtils.parseObject(responseContent); ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
if (!jsonObject.has("app")) {
return ExecutionStatus.FAILURE;
}
result = jsonObject.path("app").path("finalStatus").asText(); result = jsonObject.path("app").path("finalStatus").asText();
} else { } else {
//may be in job history //may be in job history
String jobHistoryUrl = getJobHistoryUrl(applicationId); String jobHistoryUrl = getJobHistoryUrl(applicationId);
logger.info("jobHistoryUrl={}", jobHistoryUrl); logger.info("jobHistoryUrl={}", jobHistoryUrl);
responseContent = HttpUtils.get(jobHistoryUrl); responseContent = HttpUtils.get(jobHistoryUrl);
ObjectNode jsonObject = JSONUtils.parseObject(responseContent); if (null != responseContent) {
if (!jsonObject.has("job")){ ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
if (!jsonObject.has("job")) {
return ExecutionStatus.FAILURE;
}
result = jsonObject.path("job").path("state").asText();
} else {
return ExecutionStatus.FAILURE; return ExecutionStatus.FAILURE;
} }
result = jsonObject.path("job").path("state").asText();
} }
switch (result) { switch (result) {
@ -474,7 +482,7 @@ public class HadoopUtils implements Closeable {
/** /**
* hdfs resource dir * hdfs resource dir
* *
* @param tenantCode tenant code * @param tenantCode tenant code
* @param resourceType resource type * @param resourceType resource type
* @return hdfs resource dir * @return hdfs resource dir
*/ */
@ -679,7 +687,7 @@ public class HadoopUtils implements Closeable {
ObjectNode jsonObject = JSONUtils.parseObject(retStr); ObjectNode jsonObject = JSONUtils.parseObject(retStr);
//get ResourceManager state //get ResourceManager state
if (!jsonObject.has("clusterInfo")){ if (!jsonObject.has("clusterInfo")) {
return null; return null;
} }
return jsonObject.get("clusterInfo").path("haState").asText(); return jsonObject.get("clusterInfo").path("haState").asText();

224
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java

@ -14,9 +14,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.client.config.AuthSchemes; import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.CookieSpecs;
@ -30,148 +32,148 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException; import java.io.IOException;
import java.security.KeyManagementException; import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.util.Arrays; import java.util.Arrays;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* http utils * http utils
*/ */
public class HttpUtils { public class HttpUtils {
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); private HttpUtils() {
throw new UnsupportedOperationException("Construct HttpUtils");
private HttpUtils() { }
}
public static CloseableHttpClient getInstance(){
return HttpClientInstance.httpClient;
}
private static class HttpClientInstance{
private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
}
private static PoolingHttpClientConnectionManager cm;
private static SSLContext ctx = null;
private static SSLConnectionSocketFactory socketFactory;
private static RequestConfig requestConfig;
private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
private static X509TrustManager xtm = new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) {
}
@Override public static CloseableHttpClient getInstance() {
public X509Certificate[] getAcceptedIssuers() { return HttpClientInstance.httpClient;
return null; }
}
};
static { private static class HttpClientInstance {
try { private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build();
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); }
ctx.init(null, new TrustManager[] { xtm }, null);
} catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) {
logger.error("SSLContext init with KeyManagementException", e);
}
socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
/** set timeout、request time、socket timeout */
requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setExpectContinueEnabled(Boolean.TRUE)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
.build();
socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setDefaultMaxPerRoute(60);
cm.setMaxTotal(100);
}
private static PoolingHttpClientConnectionManager cm;
private static SSLContext ctx = null;
private static SSLConnectionSocketFactory socketFactory;
private static RequestConfig requestConfig;
private static Registry<ConnectionSocketFactory> socketFactoryRegistry;
private static X509TrustManager xtm = new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
static {
try {
ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS);
ctx.init(null, new TrustManager[]{xtm}, null);
} catch (NoSuchAlgorithmException e) {
logger.error("SSLContext init with NoSuchAlgorithmException", e);
} catch (KeyManagementException e) {
logger.error("SSLContext init with KeyManagementException", e);
}
socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE);
/** set timeout、request time、socket timeout */
requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.setExpectContinueEnabled(Boolean.TRUE)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT)
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true)
.build();
socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build();
cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
cm.setDefaultMaxPerRoute(60);
cm.setMaxTotal(100);
}
/** /**
* get http request content * get http request content
* @param url url *
* @return http get request response content * @param url url
*/ * @return http get request response content
public static String get(String url){ */
CloseableHttpClient httpclient = HttpUtils.getInstance(); public static String get(String url) {
CloseableHttpClient httpclient = HttpUtils.getInstance();
HttpGet httpget = new HttpGet(url); HttpGet httpget = new HttpGet(url);
return getResponseContentString(httpget,httpclient); return getResponseContentString(httpget, httpclient);
} }
/** /**
* get http response content * get http response content
* *
* @param httpget httpget * @param httpget httpget
* @param httpClient httpClient * @param httpClient httpClient
* @return http get request response content * @return http get request response content
*/ */
public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) { public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
String responseContent = null; String responseContent = null;
CloseableHttpResponse response = null; CloseableHttpResponse response = null;
try { try {
response = httpClient.execute(httpget); response = httpClient.execute(httpget);
// check response status is 200 // check response status is 200
if (response.getStatusLine().getStatusCode() == 200) { if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity(); HttpEntity entity = response.getEntity();
if (entity != null) { if (entity != null) {
responseContent = EntityUtils.toString(entity, Constants.UTF_8); responseContent = EntityUtils.toString(entity, Constants.UTF_8);
} else { } else {
logger.warn("http entity is null"); logger.warn("http entity is null");
} }
} else { } else {
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode()); logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
} }
} catch (IOException ioe) { } catch (IOException ioe) {
logger.error(ioe.getMessage(), ioe); logger.error(ioe.getMessage(), ioe);
} finally { } finally {
try { try {
if (response != null) { if (response != null) {
EntityUtils.consume(response.getEntity()); EntityUtils.consume(response.getEntity());
response.close(); response.close();
} }
} catch (IOException e) { } catch (IOException e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
if (!httpget.isAborted()) { if (!httpget.isAborted()) {
httpget.releaseConnection(); httpget.releaseConnection();
httpget.abort(); httpget.abort();
} }
} }
return responseContent; return responseContent;
} }
} }

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java

@ -1,4 +1,3 @@
/* /*
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
@ -18,14 +17,17 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
public class IOUtils { public class IOUtils {
public static void closeQuietly(Closeable closeable){ private IOUtils() {
if(closeable != null){ throw new UnsupportedOperationException("Construct IOUtils");
}
public static void closeQuietly(Closeable closeable) {
if (closeable != null) {
try { try {
closeable.close(); closeable.close();
} catch (IOException ignore) { } catch (IOException ignore) {

72
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java

@ -14,46 +14,50 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
/** /**
* http utils * http utils
*/ */
public class IpUtils { public class IpUtils {
public static final String DOT = "."; private IpUtils() {
throw new UnsupportedOperationException("Construct IpUtils");
/** }
* ip str to long <p>
* public static final String DOT = ".";
* @param ipStr ip string
* @return ip to long /**
*/ * ip str to long <p>
public static Long ipToLong(String ipStr) { *
String[] ipSet = ipStr.split("\\" + DOT); * @param ipStr ip string
* @return ip to long
return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]); */
} public static Long ipToLong(String ipStr) {
String[] ipSet = ipStr.split("\\" + DOT);
/**
* long to ip return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
* @param ipLong the long number converted from IP }
* @return String
*/ /**
public static String longToIp(long ipLong) { * long to ip
long[] ipNumbers = new long[4]; *
long tmp = 0xFF; * @param ipLong the long number converted from IP
ipNumbers[0] = ipLong >> 24 & tmp; * @return String
ipNumbers[1] = ipLong >> 16 & tmp; */
ipNumbers[2] = ipLong >> 8 & tmp; public static String longToIp(long ipLong) {
ipNumbers[3] = ipLong & tmp; long[] ipNumbers = new long[4];
long tmp = 0xFF;
String sb = ipNumbers[0] + DOT + ipNumbers[0] = ipLong >> 24 & tmp;
ipNumbers[1] + DOT + ipNumbers[1] = ipLong >> 16 & tmp;
ipNumbers[2] + DOT + ipNumbers[2] = ipLong >> 8 & tmp;
ipNumbers[3]; ipNumbers[3] = ipLong & tmp;
return sb;
} return ipNumbers[0] + DOT
+ ipNumbers[1] + DOT
+ ipNumbers[2] + DOT
+ ipNumbers[3];
}
} }

62
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java

@ -14,25 +14,39 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.node.TextNode;
import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.databind.type.CollectionType;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
import static com.fasterxml.jackson.databind.DeserializationFeature.*;
import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS;
/** /**
* json utils * json utils
@ -49,13 +63,12 @@ public class JSONUtils {
.configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
.configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true)
.configure(REQUIRE_SETTERS_FOR_GETTERS, true) .configure(REQUIRE_SETTERS_FOR_GETTERS, true)
.setTimeZone(TimeZone.getDefault()) .setTimeZone(TimeZone.getDefault());
;
private JSONUtils() { private JSONUtils() {
throw new UnsupportedOperationException("Construct JSONUtils");
} }
public static ArrayNode createArrayNode() { public static ArrayNode createArrayNode() {
return objectMapper.createArrayNode(); return objectMapper.createArrayNode();
} }
@ -94,9 +107,9 @@ public class JSONUtils {
* the fields of the specified object are generics, just the object itself should not be a * the fields of the specified object are generics, just the object itself should not be a
* generic type. * generic type.
* *
* @param json the string from which the object is to be deserialized * @param json the string from which the object is to be deserialized
* @param clazz the class of T * @param clazz the class of T
* @param <T> T * @param <T> T
* @return an object of type T from the string * @return an object of type T from the string
* classOfT * classOfT
*/ */
@ -116,9 +129,9 @@ public class JSONUtils {
/** /**
* json to list * json to list
* *
* @param json json string * @param json json string
* @param clazz class * @param clazz class
* @param <T> T * @param <T> T
* @return list * @return list
*/ */
public static <T> List<T> toList(String json, Class<T> clazz) { public static <T> List<T> toList(String json, Class<T> clazz) {
@ -137,7 +150,6 @@ public class JSONUtils {
return Collections.emptyList(); return Collections.emptyList();
} }
/** /**
* check json object valid * check json object valid
* *
@ -160,13 +172,12 @@ public class JSONUtils {
return false; return false;
} }
/** /**
* Method for finding a JSON Object field with specified name in this * Method for finding a JSON Object field with specified name in this
* node or its child nodes, and returning value it has. * node or its child nodes, and returning value it has.
* If no matching field is found in this node or its descendants, returns null. * If no matching field is found in this node or its descendants, returns null.
* *
* @param jsonNode json node * @param jsonNode json node
* @param fieldName Name of field to look for * @param fieldName Name of field to look for
* @return Value of first matching node found, if any; null if none * @return Value of first matching node found, if any; null if none
*/ */
@ -180,7 +191,6 @@ public class JSONUtils {
return node.toString(); return node.toString();
} }
/** /**
* json to map * json to map
* <p> * <p>
@ -195,7 +205,8 @@ public class JSONUtils {
} }
try { try {
return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {}); return objectMapper.readValue(json, new TypeReference<Map<String, String>>() {
});
} catch (Exception e) { } catch (Exception e) {
logger.error("json to map exception!", e); logger.error("json to map exception!", e);
} }
@ -206,11 +217,11 @@ public class JSONUtils {
/** /**
* json to map * json to map
* *
* @param json json * @param json json
* @param classK classK * @param classK classK
* @param classV classV * @param classV classV
* @param <K> K * @param <K> K
* @param <V> V * @param <V> V
* @return to map * @return to map
*/ */
public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) { public static <K, V> Map<K, V> toMap(String json, Class<K> classK, Class<V> classV) {
@ -258,7 +269,6 @@ public class JSONUtils {
} }
} }
/** /**
* json serializer * json serializer
*/ */

32
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java

@ -14,22 +14,28 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger; import org.slf4j.Logger;
/** /**
* logger utils * logger utils
*/ */
public class LoggerUtils { public class LoggerUtils {
private LoggerUtils() {
throw new UnsupportedOperationException("Construct LoggerUtils");
}
/** /**
* rules for extracting application ID * rules for extracting application ID
*/ */
@ -53,29 +59,29 @@ public class LoggerUtils {
/** /**
* build job id * build job id
* *
* @param affix Task Logger's prefix * @param affix Task Logger's prefix
* @param processDefId process define id * @param processDefId process define id
* @param processInstId process instance id * @param processInstId process instance id
* @param taskId task id * @param taskId task id
* @return task id format * @return task id format
*/ */
public static String buildTaskId(String affix, public static String buildTaskId(String affix,
int processDefId, int processDefId,
int processInstId, int processInstId,
int taskId){ int taskId) {
// - [taskAppId=TASK_79_4084_15210] // - [taskAppId=TASK_79_4084_15210]
return String.format(" - %s%s-%s-%s-%s]",TASK_APPID_LOG_FORMAT,affix, return String.format(" - %s%s-%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix,
processDefId, processDefId,
processInstId, processInstId,
taskId); taskId);
} }
/** /**
* processing log * processing log
* get yarn application id list * get yarn application id list
* @param log log content *
* @param logger logger * @param log log content
* @param logger logger
* @return app id list * @return app id list
*/ */
public static List<String> getAppIds(String log, Logger logger) { public static List<String> getAppIds(String log, Logger logger) {
@ -87,7 +93,7 @@ public class LoggerUtils {
// analyse logs to get all submit yarn application id // analyse logs to get all submit yarn application id
while (matcher.find()) { while (matcher.find()) {
String appId = matcher.group(); String appId = matcher.group();
if(!appIds.contains(appId)){ if (!appIds.contains(appId)) {
logger.info("find app id: {}", appId); logger.info("find app id: {}", appId);
appIds.add(appId); appIds.add(appId);
} }

25
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java

@ -14,27 +14,36 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger; import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE;
import org.slf4j.LoggerFactory;
import static java.util.Collections.emptyList;
import java.io.IOException; import java.io.IOException;
import java.net.*; import java.net.Inet6Address;
import java.util.*; import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static java.util.Collections.emptyList; import org.slf4j.Logger;
import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE; import org.slf4j.LoggerFactory;
/** /**
* NetUtils * NetUtils
*/ */
public class NetUtils { public class NetUtils {
private NetUtils() { private NetUtils() {
throw new IllegalStateException("Utility class"); throw new UnsupportedOperationException("Construct NetUtils");
} }
private static Logger logger = LoggerFactory.getLogger(NetUtils.class); private static Logger logger = LoggerFactory.getLogger(NetUtils.class);

818
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@ -14,8 +14,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.configuration.Configuration;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
@ -32,9 +38,6 @@ import java.util.Optional;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.configuration.Configuration;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -45,431 +48,444 @@ import oshi.hardware.HardwareAbstractionLayer;
/** /**
* os utils * os utils
*
*/ */
public class OSUtils { public class OSUtils {
private static final Logger logger = LoggerFactory.getLogger(OSUtils.class); private static final Logger logger = LoggerFactory.getLogger(OSUtils.class);
public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>(); public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
private static final SystemInfo SI = new SystemInfo(); private static final SystemInfo SI = new SystemInfo();
public static final String TWO_DECIMAL = "0.00"; public static final String TWO_DECIMAL = "0.00";
/** /**
* return -1 when the function can not get hardware env info * return -1 when the function can not get hardware env info
* e.g {@link OSUtils#loadAverage()} {@link OSUtils#cpuUsage()} * e.g {@link OSUtils#loadAverage()} {@link OSUtils#cpuUsage()}
*/ */
public static final double NEGATIVE_ONE = -1; public static final double NEGATIVE_ONE = -1;
private static HardwareAbstractionLayer hal = SI.getHardware(); private static HardwareAbstractionLayer hal = SI.getHardware();
private OSUtils() {} private OSUtils() {
throw new UnsupportedOperationException("Construct OSUtils");
/**
* Initialization regularization, solve the problem of pre-compilation performance,
* avoid the thread safety problem of multi-thread operation
*/
private static final Pattern PATTERN = Pattern.compile("\\s+");
/**
* get memory usage
* Keep 2 decimal
* @return percent %
*/
public static double memoryUsage() {
GlobalMemory memory = hal.getMemory();
double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(memoryUsage));
}
/**
* get available physical memory size
*
* Keep 2 decimal
* @return available Physical Memory Size, unit: G
*/
public static double availablePhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) /1024.0/1024/1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
}
/**
* get total physical memory size
*
* Keep 2 decimal
* @return available Physical Memory Size, unit: G
*/
public static double totalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = memory.getTotal() /1024.0/1024/1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
}
/**
* load average
*
* @return load average
*/
public static double loadAverage() {
double loadAverage = hal.getProcessor().getSystemLoadAverage();
if (Double.isNaN(loadAverage)) {
return NEGATIVE_ONE;
} }
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); /**
df.setRoundingMode(RoundingMode.HALF_UP); * Initialization regularization, solve the problem of pre-compilation performance,
return Double.parseDouble(df.format(loadAverage)); * avoid the thread safety problem of multi-thread operation
} */
private static final Pattern PATTERN = Pattern.compile("\\s+");
/**
* get cpu usage /**
* * get memory usage
* @return cpu usage * Keep 2 decimal
*/ *
public static double cpuUsage() { * @return percent %
CentralProcessor processor = hal.getProcessor(); */
double cpuUsage = processor.getSystemCpuLoad(); public static double memoryUsage() {
if (Double.isNaN(cpuUsage)) { GlobalMemory memory = hal.getMemory();
return NEGATIVE_ONE; double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(memoryUsage));
} }
DecimalFormat df = new DecimalFormat(TWO_DECIMAL); /**
df.setRoundingMode(RoundingMode.HALF_UP); * get available physical memory size
return Double.parseDouble(df.format(cpuUsage)); * <p>
} * Keep 2 decimal
*
public static List<String> getUserList() { * @return available Physical Memory Size, unit: G
try { */
if (isMacOS()) { public static double availablePhysicalMemorySize() {
return getUserListFromMac(); GlobalMemory memory = hal.getMemory();
} else if (isWindows()) { double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024;
return getUserListFromWindows();
} else { DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
return getUserListFromLinux(); df.setRoundingMode(RoundingMode.HALF_UP);
} return Double.parseDouble(df.format(availablePhysicalMemorySize));
} catch (Exception e) {
logger.error(e.getMessage(), e); }
/**
* get total physical memory size
* <p>
* Keep 2 decimal
*
* @return available Physical Memory Size, unit: G
*/
public static double totalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
}
/**
* load average
*
* @return load average
*/
public static double loadAverage() {
double loadAverage = hal.getProcessor().getSystemLoadAverage();
if (Double.isNaN(loadAverage)) {
return NEGATIVE_ONE;
}
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(loadAverage));
} }
return Collections.emptyList(); /**
} * get cpu usage
*
/** * @return cpu usage
* get user list from linux */
* public static double cpuUsage() {
* @return user list CentralProcessor processor = hal.getProcessor();
*/ double cpuUsage = processor.getSystemCpuLoad();
private static List<String> getUserListFromLinux() throws IOException { if (Double.isNaN(cpuUsage)) {
List<String> userList = new ArrayList<>(); return NEGATIVE_ONE;
try (BufferedReader bufferedReader = new BufferedReader(
new InputStreamReader(new FileInputStream("/etc/passwd")))) {
String line;
while ((line = bufferedReader.readLine()) != null) {
if (line.contains(":")) {
String[] userInfo = line.split(":");
userList.add(userInfo[0]);
} }
}
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(cpuUsage));
} }
return userList; public static List<String> getUserList() {
} try {
if (isMacOS()) {
/** return getUserListFromMac();
* get user list from mac } else if (isWindows()) {
* @return user list return getUserListFromWindows();
*/ } else {
private static List<String> getUserListFromMac() throws IOException { return getUserListFromLinux();
String result = exeCmd("dscl . list /users"); }
if (StringUtils.isNotEmpty(result)) { } catch (Exception e) {
return Arrays.asList(result.split( "\n")); logger.error(e.getMessage(), e);
}
return Collections.emptyList();
}
/**
* get user list from linux
*
* @return user list
*/
private static List<String> getUserListFromLinux() throws IOException {
List<String> userList = new ArrayList<>();
try (BufferedReader bufferedReader = new BufferedReader(
new InputStreamReader(new FileInputStream("/etc/passwd")))) {
String line;
while ((line = bufferedReader.readLine()) != null) {
if (line.contains(":")) {
String[] userInfo = line.split(":");
userList.add(userInfo[0]);
}
}
}
return userList;
} }
return Collections.emptyList(); /**
} * get user list from mac
*
/** * @return user list
* get user list from windows */
* @return user list private static List<String> getUserListFromMac() throws IOException {
* @throws IOException String result = exeCmd("dscl . list /users");
*/ if (StringUtils.isNotEmpty(result)) {
private static List<String> getUserListFromWindows() throws IOException { return Arrays.asList(result.split("\n"));
String result = exeCmd("net user");
String[] lines = result.split("\n");
int startPos = 0;
int endPos = lines.length - 2;
for (int i = 0; i < lines.length; i++) {
if (lines[i].isEmpty()) {
continue;
}
int count = 0;
if (lines[i].charAt(0) == '-') {
for (int j = 0; j < lines[i].length(); j++) {
if (lines[i].charAt(i) == '-') {
count++;
}
} }
}
if (count == lines[i].length()) { return Collections.emptyList();
startPos = i + 1;
break;
}
} }
List<String> users = new ArrayList<>(); /**
while (startPos <= endPos) { * get user list from windows
users.addAll(Arrays.asList(PATTERN.split(lines[startPos]))); *
startPos++; * @return user list
*/
private static List<String> getUserListFromWindows() throws IOException {
String result = exeCmd("net user");
String[] lines = result.split("\n");
int startPos = 0;
int endPos = lines.length - 2;
for (int i = 0; i < lines.length; i++) {
if (lines[i].isEmpty()) {
continue;
}
int count = 0;
if (lines[i].charAt(0) == '-') {
for (int j = 0; j < lines[i].length(); j++) {
if (lines[i].charAt(i) == '-') {
count++;
}
}
}
if (count == lines[i].length()) {
startPos = i + 1;
break;
}
}
List<String> users = new ArrayList<>();
while (startPos <= endPos) {
users.addAll(Arrays.asList(PATTERN.split(lines[startPos])));
startPos++;
}
return users;
} }
return users; /**
} * create user
*
/** * @param userName user name
* create user * @return true if creation was successful, otherwise false
* @param userName user name */
* @return true if creation was successful, otherwise false public static boolean createUser(String userName) {
*/ try {
public static boolean createUser(String userName) { String userGroup = OSUtils.getGroup();
try { if (StringUtils.isEmpty(userGroup)) {
String userGroup = OSUtils.getGroup(); String errorLog = String.format("%s group does not exist for this operating system.", userGroup);
if (StringUtils.isEmpty(userGroup)) { LoggerUtils.logError(Optional.ofNullable(logger), errorLog);
String errorLog = String.format("%s group does not exist for this operating system.", userGroup); LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog);
LoggerUtils.logError(Optional.ofNullable(logger), errorLog); return false;
LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog); }
if (isMacOS()) {
createMacUser(userName, userGroup);
} else if (isWindows()) {
createWindowsUser(userName, userGroup);
} else {
createLinuxUser(userName, userGroup);
}
return true;
} catch (Exception e) {
LoggerUtils.logError(Optional.ofNullable(logger), e);
LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
}
return false; return false;
}
if (isMacOS()) {
createMacUser(userName, userGroup);
} else if (isWindows()) {
createWindowsUser(userName, userGroup);
} else {
createLinuxUser(userName, userGroup);
}
return true;
} catch (Exception e) {
LoggerUtils.logError(Optional.ofNullable(logger), e);
LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
} }
return false; /**
} * create linux user
*
/** * @param userName user name
* create linux user * @param userGroup user group
* @param userName user name * @throws IOException in case of an I/O error
* @param userGroup user group */
* @throws IOException in case of an I/O error private static void createLinuxUser(String userName, String userGroup) throws IOException {
*/ String infoLog1 = String.format("create linux os user : %s", userName);
private static void createLinuxUser(String userName, String userGroup) throws IOException { LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
String infoLog1 = String.format("create linux os user : %s", userName); LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1); String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
String infoLog2 = String.format("execute cmd : %s", cmd);
String cmd = String.format("sudo useradd -g %s %s", userGroup, userName); LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
String infoLog2 = String.format("execute cmd : %s", cmd); LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); OSUtils.exeCmd(cmd);
LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
OSUtils.exeCmd(cmd);
}
/**
* create mac user (Supports Mac OSX 10.10+)
* @param userName user name
* @param userGroup user group
* @throws IOException in case of an I/O error
*/
private static void createMacUser(String userName, String userGroup) throws IOException {
Optional<Logger> optionalLogger = Optional.ofNullable(logger);
Optional<Logger> optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get());
String infoLog1 = String.format("create mac os user : %s", userName);
LoggerUtils.logInfo(optionalLogger, infoLog1);
LoggerUtils.logInfo(optionalTaskLogger, infoLog1);
String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
String infoLog2 = String.format("create user command : %s", createUserCmd);
LoggerUtils.logInfo(optionalLogger, infoLog2);
LoggerUtils.logInfo(optionalTaskLogger, infoLog2);
OSUtils.exeCmd(createUserCmd);
String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
String infoLog3 = String.format("append user to group : %s", appendGroupCmd);
LoggerUtils.logInfo(optionalLogger, infoLog3);
LoggerUtils.logInfo(optionalTaskLogger, infoLog3);
OSUtils.exeCmd(appendGroupCmd);
}
/**
* create windows user
* @param userName user name
* @param userGroup user group
* @throws IOException in case of an I/O error
*/
private static void createWindowsUser(String userName, String userGroup) throws IOException {
String infoLog1 = String.format("create windows os user : %s", userName);
LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
String userCreateCmd = String.format("net user \"%s\" /add", userName);
String infoLog2 = String.format("execute create user command : %s", userCreateCmd);
LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
OSUtils.exeCmd(userCreateCmd);
String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd);
LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3);
LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3);
OSUtils.exeCmd(appendGroupCmd);
}
/**
* get system group information
* @return system group info
* @throws IOException errors
*/
public static String getGroup() throws IOException {
if (isWindows()) {
String currentProcUserName = System.getProperty("user.name");
String result = exeCmd(String.format("net user \"%s\"", currentProcUserName));
String line = result.split("\n")[22];
String group = PATTERN.split(line)[1];
if (group.charAt(0) == '*') {
return group.substring(1);
} else {
return group;
}
} else {
String result = exeCmd("groups");
if (StringUtils.isNotEmpty(result)) {
String[] groupInfo = result.split(" ");
return groupInfo[0];
}
} }
return null; /**
} * create mac user (Supports Mac OSX 10.10+)
*
/** * @param userName user name
* Execute the corresponding command of Linux or Windows * @param userGroup user group
* * @throws IOException in case of an I/O error
* @param command command */
* @return result of execute command private static void createMacUser(String userName, String userGroup) throws IOException {
* @throws IOException errors
*/ Optional<Logger> optionalLogger = Optional.ofNullable(logger);
public static String exeCmd(String command) throws IOException { Optional<Logger> optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get());
StringTokenizer st = new StringTokenizer(command);
String[] cmdArray = new String[st.countTokens()]; String infoLog1 = String.format("create mac os user : %s", userName);
for (int i = 0; st.hasMoreTokens(); i++) { LoggerUtils.logInfo(optionalLogger, infoLog1);
cmdArray[i] = st.nextToken(); LoggerUtils.logInfo(optionalTaskLogger, infoLog1);
String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
String infoLog2 = String.format("create user command : %s", createUserCmd);
LoggerUtils.logInfo(optionalLogger, infoLog2);
LoggerUtils.logInfo(optionalTaskLogger, infoLog2);
OSUtils.exeCmd(createUserCmd);
String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
String infoLog3 = String.format("append user to group : %s", appendGroupCmd);
LoggerUtils.logInfo(optionalLogger, infoLog3);
LoggerUtils.logInfo(optionalTaskLogger, infoLog3);
OSUtils.exeCmd(appendGroupCmd);
} }
return exeShell(cmdArray);
} /**
* create windows user
/** *
* Execute the shell * @param userName user name
* @param command command * @param userGroup user group
* @return result of execute the shell * @throws IOException in case of an I/O error
* @throws IOException errors */
*/ private static void createWindowsUser(String userName, String userGroup) throws IOException {
public static String exeShell(String[] command) throws IOException { String infoLog1 = String.format("create windows os user : %s", userName);
return ShellExecutor.execCommand(command); LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
} LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
/** String userCreateCmd = String.format("net user \"%s\" /add", userName);
* get process id String infoLog2 = String.format("execute create user command : %s", userCreateCmd);
* @return process id LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
*/ LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
public static int getProcessID() { OSUtils.exeCmd(userCreateCmd);
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
return Integer.parseInt(runtimeMXBean.getName().split("@")[0]); String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
} String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd);
LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3);
/** LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3);
* whether is macOS OSUtils.exeCmd(appendGroupCmd);
* @return true if mac
*/
public static boolean isMacOS() {
return getOSName().startsWith("Mac");
}
/**
* whether is windows
* @return true if windows
*/
public static boolean isWindows() {
return getOSName().startsWith("Windows");
}
/**
* get current OS name
* @return current OS name
*/
public static String getOSName() {
return System.getProperty("os.name");
}
/**
* check memory and cpu usage
* @param systemCpuLoad systemCpuLoad
* @param systemReservedMemory systemReservedMemory
* @return check memory and cpu usage
*/
public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){
// system load average
double loadAverage = OSUtils.loadAverage();
// system available physical memory
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){
logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage);
return false;
}else{
return true;
} }
}
/**
/** * get system group information
* check memory and cpu usage *
* @param conf conf * @return system group info
* @param isMaster is master * @throws IOException errors
* @return check memory and cpu usage */
*/ public static String getGroup() throws IOException {
public static Boolean checkResource(Configuration conf, Boolean isMaster){ if (isWindows()) {
double systemCpuLoad; String currentProcUserName = System.getProperty("user.name");
double systemReservedMemory; String result = exeCmd(String.format("net user \"%s\"", currentProcUserName));
String line = result.split("\n")[22];
if(Boolean.TRUE.equals(isMaster)){ String group = PATTERN.split(line)[1];
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); if (group.charAt(0) == '*') {
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); return group.substring(1);
}else{ } else {
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); return group;
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); }
} else {
String result = exeCmd("groups");
if (StringUtils.isNotEmpty(result)) {
String[] groupInfo = result.split(" ");
return groupInfo[0];
}
}
return null;
}
/**
* Execute the corresponding command of Linux or Windows
*
* @param command command
* @return result of execute command
* @throws IOException errors
*/
public static String exeCmd(String command) throws IOException {
StringTokenizer st = new StringTokenizer(command);
String[] cmdArray = new String[st.countTokens()];
for (int i = 0; st.hasMoreTokens(); i++) {
cmdArray[i] = st.nextToken();
}
return exeShell(cmdArray);
}
/**
* Execute the shell
*
* @param command command
* @return result of execute the shell
* @throws IOException errors
*/
public static String exeShell(String[] command) throws IOException {
return ShellExecutor.execCommand(command);
}
/**
* get process id
*
* @return process id
*/
public static int getProcessID() {
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
return Integer.parseInt(runtimeMXBean.getName().split("@")[0]);
}
/**
* whether is macOS
*
* @return true if mac
*/
public static boolean isMacOS() {
return getOSName().startsWith("Mac");
}
/**
* whether is windows
*
* @return true if windows
*/
public static boolean isWindows() {
return getOSName().startsWith("Windows");
}
/**
* get current OS name
*
* @return current OS name
*/
public static String getOSName() {
return System.getProperty("os.name");
}
/**
* check memory and cpu usage
*
* @param systemCpuLoad systemCpuLoad
* @param systemReservedMemory systemReservedMemory
* @return check memory and cpu usage
*/
public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory) {
// system load average
double loadAverage = OSUtils.loadAverage();
// system available physical memory
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
if (loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory) {
logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage);
return false;
} else {
return true;
}
}
/**
* check memory and cpu usage
*
* @param conf conf
* @param isMaster is master
* @return check memory and cpu usage
*/
public static Boolean checkResource(Configuration conf, Boolean isMaster) {
double systemCpuLoad;
double systemReservedMemory;
if (Boolean.TRUE.equals(isMaster)) {
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY);
} else {
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY);
}
return checkResource(systemCpuLoad, systemReservedMemory);
} }
return checkResource(systemCpuLoad,systemReservedMemory);
}
} }

406
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java

@ -14,10 +14,9 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.DataType; import org.apache.dolphinscheduler.common.enums.DataType;
@ -25,231 +24,228 @@ import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.text.ParseException; import java.util.Date;
import java.util.*; import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* parameter parse utils * parameter parse utils
*/ */
public class ParameterUtils { public class ParameterUtils {
private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
/**
* convert parameters place holders
*
* @param parameterString parameter
* @param parameterMap parameter map
* @return convert parameters place holders
*/
public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
if (StringUtils.isEmpty(parameterString) || parameterMap == null) {
return parameterString;
}
//Get current time, schedule execute time
String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
Date cronTime = null;
if (StringUtils.isNotEmpty(cronTimeStr)) {
try {
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME});
} catch (ParseException e) {
logger.error("parse {} exception", cronTimeStr, e);
}
} else {
cronTime = new Date();
}
// replace variable ${} form,refers to the replacement of system variables and custom variables
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
// replace time $[...] form, eg. $[yyyyMMdd] private ParameterUtils() {
if (cronTime != null) { throw new UnsupportedOperationException("Construct ParameterUtils");
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
} }
return parameterString; /**
} * convert parameters place holders
*
/** * @param parameterString parameter
* new * @param parameterMap parameter map
* convert parameters place holders * @return convert parameters place holders
* */
* @param parameterString parameter public static String convertParameterPlaceholders(String parameterString, Map<String, String> parameterMap) {
* @param parameterMap parameter map if (StringUtils.isEmpty(parameterString) || parameterMap == null) {
* @return convert parameters place holders return parameterString;
*/ }
public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
if (StringUtils.isEmpty(parameterString)) { //Get current time, schedule execute time
return parameterString; String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME);
}
//Get current time, schedule execute time Date cronTime = null;
String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
Date cronTime = null; if (StringUtils.isNotEmpty(cronTimeStr)) {
cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
if (StringUtils.isNotEmpty(cronTimeStr)) { } else {
try { cronTime = new Date();
cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME}); }
} catch (ParseException e) { // replace variable ${} form,refers to the replacement of system variables and custom variables
logger.error(String.format("parse %s exception", cronTimeStr), e); parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
}
} else { // replace time $[...] form, eg. $[yyyyMMdd]
cronTime = new Date(); if (cronTime != null) {
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
}
return parameterString;
} }
// replace variable ${} form,refers to the replacement of system variables and custom variables /**
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); * new
* convert parameters place holders
// replace time $[...] form, eg. $[yyyyMMdd] *
if (cronTime != null) { * @param parameterString parameter
parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true); * @param parameterMap parameter map
* @return convert parameters place holders
} */
return parameterString; public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
} if (StringUtils.isEmpty(parameterString)) {
return parameterString;
}
/** //Get current time, schedule execute time
* set in parameter String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
* @param index index Date cronTime = null;
* @param stmt preparedstatement
* @param dataType data type if (StringUtils.isNotEmpty(cronTimeStr)) {
* @param value value cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
* @throws Exception errors
*/ } else {
public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{ cronTime = new Date();
if (dataType.equals(DataType.VARCHAR)){ }
stmt.setString(index,value);
}else if (dataType.equals(DataType.INTEGER)){ // replace variable ${} form,refers to the replacement of system variables and custom variables
stmt.setInt(index, Integer.parseInt(value)); parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
}else if (dataType.equals(DataType.LONG)){
stmt.setLong(index, Long.parseLong(value)); // replace time $[...] form, eg. $[yyyyMMdd]
}else if (dataType.equals(DataType.FLOAT)){ if (cronTime != null) {
stmt.setFloat(index, Float.parseFloat(value)); parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true);
}else if (dataType.equals(DataType.DOUBLE)){
stmt.setDouble(index, Double.parseDouble(value)); }
}else if (dataType.equals(DataType.DATE)){ return parameterString;
stmt.setDate(index, java.sql.Date.valueOf(value));
}else if (dataType.equals(DataType.TIME)){
stmt.setString(index, value);
}else if (dataType.equals(DataType.TIMESTAMP)){
stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
}else if (dataType.equals(DataType.BOOLEAN)){
stmt.setBoolean(index,Boolean.parseBoolean(value));
}
}
/**
* curing user define parameters
*
* @param globalParamMap global param map
* @param globalParamList global param list
* @param commandType command type
* @param scheduleTime schedule time
* @return curing user define parameters
*/
public static String curingGlobalParams(Map<String,String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime){
if (globalParamList == null || globalParamList.isEmpty()) {
return null;
} }
Map<String, String> globalMap = new HashMap<>(); /**
if (globalParamMap!= null){ * set in parameter
globalMap.putAll(globalParamMap); *
* @param index index
* @param stmt preparedstatement
* @param dataType data type
* @param value value
* @throws Exception errors
*/
public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception {
if (dataType.equals(DataType.VARCHAR)) {
stmt.setString(index, value);
} else if (dataType.equals(DataType.INTEGER)) {
stmt.setInt(index, Integer.parseInt(value));
} else if (dataType.equals(DataType.LONG)) {
stmt.setLong(index, Long.parseLong(value));
} else if (dataType.equals(DataType.FLOAT)) {
stmt.setFloat(index, Float.parseFloat(value));
} else if (dataType.equals(DataType.DOUBLE)) {
stmt.setDouble(index, Double.parseDouble(value));
} else if (dataType.equals(DataType.DATE)) {
stmt.setDate(index, java.sql.Date.valueOf(value));
} else if (dataType.equals(DataType.TIME)) {
stmt.setString(index, value);
} else if (dataType.equals(DataType.TIMESTAMP)) {
stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
} else if (dataType.equals(DataType.BOOLEAN)) {
stmt.setBoolean(index, Boolean.parseBoolean(value));
}
} }
Map<String,String> allParamMap = new HashMap<>();
//If it is a complement, a complement time needs to be passed in, according to the task type
Map<String,String> timeParams = BusinessTimeUtils
.getBusinessTime(commandType, scheduleTime);
if (timeParams != null) { /**
allParamMap.putAll(timeParams); * curing user define parameters
*
* @param globalParamMap global param map
* @param globalParamList global param list
* @param commandType command type
* @param scheduleTime schedule time
* @return curing user define parameters
*/
public static String curingGlobalParams(Map<String, String> globalParamMap, List<Property> globalParamList,
CommandType commandType, Date scheduleTime) {
if (globalParamList == null || globalParamList.isEmpty()) {
return null;
}
Map<String, String> globalMap = new HashMap<>();
if (globalParamMap != null) {
globalMap.putAll(globalParamMap);
}
Map<String, String> allParamMap = new HashMap<>();
//If it is a complement, a complement time needs to be passed in, according to the task type
Map<String, String> timeParams = BusinessTimeUtils
.getBusinessTime(commandType, scheduleTime);
if (timeParams != null) {
allParamMap.putAll(timeParams);
}
allParamMap.putAll(globalMap);
Set<Map.Entry<String, String>> entries = allParamMap.entrySet();
Map<String, String> resolveMap = new HashMap<>();
for (Map.Entry<String, String> entry : entries) {
String val = entry.getValue();
if (val.startsWith("$")) {
String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap);
resolveMap.put(entry.getKey(), str);
}
}
globalMap.putAll(resolveMap);
for (Property property : globalParamList) {
String val = globalMap.get(property.getProp());
if (val != null) {
property.setValue(val);
}
}
return JSONUtils.toJsonString(globalParamList);
} }
allParamMap.putAll(globalMap); /**
* handle escapes
Set<Map.Entry<String, String>> entries = allParamMap.entrySet(); *
* @param inputString input string
Map<String,String> resolveMap = new HashMap<>(); * @return string filter escapes
for (Map.Entry<String,String> entry : entries){ */
String val = entry.getValue(); public static String handleEscapes(String inputString) {
if (val.startsWith("$")){
String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap); if (StringUtils.isNotEmpty(inputString)) {
resolveMap.put(entry.getKey(),str); return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_");
} }
return inputString;
} }
globalMap.putAll(resolveMap);
for (Property property : globalParamList){ /**
String val = globalMap.get(property.getProp()); * $[yyyyMMdd] replace schedule time
if (val != null){ */
property.setValue(val); public static String replaceScheduleTime(String text, Date scheduleTime) {
} Map<String, Property> paramsMap = new HashMap<>();
//if getScheduleTime null ,is current date
if (null == scheduleTime) {
scheduleTime = new Date();
}
String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
Property p = new Property();
p.setValue(dateTime);
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
return text;
} }
return JSONUtils.toJsonString(globalParamList);
}
/** /**
* handle escapes * format convert
* @param inputString input string *
* @return string filter escapes * @param paramsMap params map
*/ * @return Map of converted
public static String handleEscapes(String inputString){ * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
*/
if(StringUtils.isNotEmpty(inputString)){ public static Map<String, String> convert(Map<String, Property> paramsMap) {
return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); Map<String, String> map = new HashMap<>();
} Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
return inputString; while (iter.hasNext()) {
} Map.Entry<String, Property> en = iter.next();
map.put(en.getKey(), en.getValue().getValue());
}
/** return map;
* $[yyyyMMdd] replace schedule time
* @param text
* @param scheduleTime
* @return
*/
public static String replaceScheduleTime(String text, Date scheduleTime) {
Map<String, Property> paramsMap = new HashMap<>();
//if getScheduleTime null ,is current date
if (null == scheduleTime) {
scheduleTime = new Date();
}
String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
Property p = new Property();
p.setValue(dateTime);
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
return text;
}
/**
* format convert
* @param paramsMap params map
* @return Map of converted
* see org.apache.dolphinscheduler.server.utils.ParamUtils.convert
*/
public static Map<String,String> convert(Map<String,Property> paramsMap){
Map<String,String> map = new HashMap<>();
Iterator<Map.Entry<String, Property>> iter = paramsMap.entrySet().iterator();
while (iter.hasNext()){
Map.Entry<String, Property> en = iter.next();
map.put(en.getKey(),en.getValue().getValue());
} }
return map;
}
} }

19
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java

@ -14,16 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
package org.apache.dolphinscheduler.common.utils;
/** /**
* utility methods for validating input * utility methods for validating input
*
*/ */
public final class Preconditions { public final class Preconditions {
private Preconditions() {} private Preconditions() {
throw new UnsupportedOperationException("Construct Preconditions");
}
/** /**
* if obj is null will throw NPE * if obj is null will throw NPE
@ -41,32 +42,30 @@ public final class Preconditions {
/** /**
* if obj is null will throw NullPointerException with error message * if obj is null will throw NullPointerException with error message
*
* @param obj obj * @param obj obj
* @param errorMsg error message * @param errorMsg error message
* @param <T> T * @param <T> T
* @return T * @return T
*/ */
public static <T> T checkNotNull(T obj, String errorMsg) { public static <T> T checkNotNull(T obj, String errorMsg) {
if (obj == null) { if (obj == null) {
throw new NullPointerException(errorMsg); throw new NullPointerException(errorMsg);
} }
return obj; return obj;
} }
/** /**
* if condition is false will throw an IllegalArgumentException with the given message * if condition is false will throw an IllegalArgumentException with the given message
* *
* @param condition condition * @param condition condition
* @param errorMsg error message * @param errorMsg error message
*
* @throws IllegalArgumentException Thrown, if the condition is violated. * @throws IllegalArgumentException Thrown, if the condition is violated.
*/ */
public static void checkArgument(boolean condition, Object errorMsg) { public static void checkArgument(boolean condition, Object errorMsg) {
if (!condition) { if (!condition) {
throw new IllegalArgumentException(String.valueOf(errorMsg)); throw new IllegalArgumentException(String.valueOf(errorMsg));
} }
} }
} }

48
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@ -14,13 +14,15 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -28,7 +30,8 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* property utils * property utils
@ -44,7 +47,7 @@ public class PropertyUtils {
private static final Properties properties = new Properties(); private static final Properties properties = new Properties();
private PropertyUtils() { private PropertyUtils() {
throw new IllegalStateException("PropertyUtils class"); throw new UnsupportedOperationException("Construct PropertyUtils");
} }
static { static {
@ -68,10 +71,9 @@ public class PropertyUtils {
} }
/** /**
* * @return judge whether resource upload startup
* @return judge whether resource upload startup
*/ */
public static Boolean getResUploadStartupState(){ public static Boolean getResUploadStartupState() {
String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE);
ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType);
return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3;
@ -113,14 +115,13 @@ public class PropertyUtils {
* get property value * get property value
* *
* @param key property name * @param key property name
* @return get property int value , if key == null, then return -1 * @return get property int value , if key == null, then return -1
*/ */
public static int getInt(String key) { public static int getInt(String key) {
return getInt(key, -1); return getInt(key, -1);
} }
/** /**
*
* @param key key * @param key key
* @param defaultValue default value * @param defaultValue default value
* @return property value * @return property value
@ -134,7 +135,7 @@ public class PropertyUtils {
try { try {
return Integer.parseInt(value); return Integer.parseInt(value);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return defaultValue; return defaultValue;
} }
@ -147,7 +148,7 @@ public class PropertyUtils {
*/ */
public static boolean getBoolean(String key) { public static boolean getBoolean(String key) {
String value = properties.getProperty(key.trim()); String value = properties.getProperty(key.trim());
if(null != value){ if (null != value) {
return Boolean.parseBoolean(value); return Boolean.parseBoolean(value);
} }
@ -163,7 +164,7 @@ public class PropertyUtils {
*/ */
public static Boolean getBoolean(String key, boolean defaultValue) { public static Boolean getBoolean(String key, boolean defaultValue) {
String value = properties.getProperty(key.trim()); String value = properties.getProperty(key.trim());
if(null != value){ if (null != value) {
return Boolean.parseBoolean(value); return Boolean.parseBoolean(value);
} }
@ -172,6 +173,7 @@ public class PropertyUtils {
/** /**
* get property long value * get property long value
*
* @param key key * @param key key
* @param defaultVal default value * @param defaultVal default value
* @return property value * @return property value
@ -182,16 +184,14 @@ public class PropertyUtils {
} }
/** /**
*
* @param key key * @param key key
* @return property value * @return property value
*/ */
public static long getLong(String key) { public static long getLong(String key) {
return getLong(key,-1); return getLong(key, -1);
} }
/** /**
*
* @param key key * @param key key
* @param defaultVal default value * @param defaultVal default value
* @return property value * @return property value
@ -201,11 +201,11 @@ public class PropertyUtils {
return val == null ? defaultVal : Double.parseDouble(val); return val == null ? defaultVal : Double.parseDouble(val);
} }
/** /**
* get array * get array
* @param key property name *
* @param splitStr separator * @param key property name
* @param splitStr separator
* @return property value through array * @return property value through array
*/ */
public static String[] getArray(String key, String splitStr) { public static String[] getArray(String key, String splitStr) {
@ -217,18 +217,17 @@ public class PropertyUtils {
String[] propertyArray = value.split(splitStr); String[] propertyArray = value.split(splitStr);
return propertyArray; return propertyArray;
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
logger.info(e.getMessage(),e); logger.info(e.getMessage(), e);
} }
return new String[0]; return new String[0];
} }
/** /**
*
* @param key key * @param key key
* @param type type * @param type type
* @param defaultValue default value * @param defaultValue default value
* @param <T> T * @param <T> T
* @return get enum value * @return get enum value
*/ */
public <T extends Enum<T>> T getEnum(String key, Class<T> type, public <T extends Enum<T>> T getEnum(String key, Class<T> type,
T defaultValue) { T defaultValue) {
@ -238,6 +237,7 @@ public class PropertyUtils {
/** /**
* get all properties with specified prefix, like: fs. * get all properties with specified prefix, like: fs.
*
* @param prefix prefix to search * @param prefix prefix to search
* @return all properties with specified prefix * @return all properties with specified prefix
*/ */
@ -253,11 +253,9 @@ public class PropertyUtils {
/** /**
* *
* @param key
* @param value
*/ */
public static void setValue(String key, String value) { public static void setValue(String key, String value) {
properties.setProperty(key,value); properties.setProperty(key, value);
} }
} }

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java

@ -17,15 +17,21 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import com.github.rholder.retry.*;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
/** /**
* The Retryer util. * The Retryer util.
*/ */
@ -35,7 +41,7 @@ public class RetryerUtils {
private static Retryer<Boolean> defaultRetryerResultNoCheck; private static Retryer<Boolean> defaultRetryerResultNoCheck;
private RetryerUtils() { private RetryerUtils() {
throw new UnsupportedOperationException("Construct RetryerUtils");
} }
private static Retryer<Boolean> getDefaultRetryerResultNoCheck() { private static Retryer<Boolean> getDefaultRetryerResultNoCheck() {

222
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java

@ -14,10 +14,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils;
import org.slf4j.Logger; package org.apache.dolphinscheduler.common.utils;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
@ -29,113 +27,123 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Metadata related common classes * Metadata related common classes
*
*/ */
public class SchemaUtils { public class SchemaUtils {
private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
private static Pattern p = Pattern.compile("\\s*|\t|\r|\n"); private static Pattern p = Pattern.compile("\\s*|\t|\r|\n");
/** private SchemaUtils() {
* Gets upgradable schemas for all upgrade directories throw new UnsupportedOperationException("Construct SchemaUtils");
* @return all schema list }
*/
@SuppressWarnings("unchecked") /**
public static List<String> getAllSchemaList() { * Gets upgradable schemas for all upgrade directories
List<String> schemaDirList = new ArrayList<>(); *
File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade"); * @return all schema list
if(schemaDirArr == null || schemaDirArr.length == 0) { */
return null; @SuppressWarnings("unchecked")
} public static List<String> getAllSchemaList() {
List<String> schemaDirList = new ArrayList<>();
for(File file : schemaDirArr) { File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade");
schemaDirList.add(file.getName()); if (schemaDirArr == null || schemaDirArr.length == 0) {
} return null;
}
Collections.sort(schemaDirList , new Comparator() {
@Override for (File file : schemaDirArr) {
public int compare(Object o1 , Object o2){ schemaDirList.add(file.getName());
try { }
String dir1 = String.valueOf(o1);
String dir2 = String.valueOf(o2); Collections.sort(schemaDirList, new Comparator() {
String version1 = dir1.split("_")[0]; @Override
String version2 = dir2.split("_")[0]; public int compare(Object o1, Object o2) {
if(version1.equals(version2)) { try {
return 0; String dir1 = String.valueOf(o1);
} String dir2 = String.valueOf(o2);
String version1 = dir1.split("_")[0];
if(SchemaUtils.isAGreatVersion(version1, version2)) { String version2 = dir2.split("_")[0];
return 1; if (version1.equals(version2)) {
} return 0;
}
return -1;
if (SchemaUtils.isAGreatVersion(version1, version2)) {
} catch (Exception e) { return 1;
logger.error(e.getMessage(),e); }
throw new RuntimeException(e);
} return -1;
}
}); } catch (Exception e) {
logger.error(e.getMessage(), e);
return schemaDirList; throw new RuntimeException(e);
} }
}
/** });
* Determine whether schemaVersion is higher than version
* @param schemaVersion schema version return schemaDirList;
* @param version version }
* @return Determine whether schemaVersion is higher than version
*/ /**
public static boolean isAGreatVersion(String schemaVersion, String version) { * Determine whether schemaVersion is higher than version
if(StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { *
throw new RuntimeException("schemaVersion or version is empty"); * @param schemaVersion schema version
} * @param version version
* @return Determine whether schemaVersion is higher than version
String[] schemaVersionArr = schemaVersion.split("\\."); */
String[] versionArr = version.split("\\."); public static boolean isAGreatVersion(String schemaVersion, String version) {
int arrLength = Math.min(schemaVersionArr.length, versionArr.length); if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) {
for(int i = 0 ; i < arrLength ; i++) { throw new RuntimeException("schemaVersion or version is empty");
if(Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) { }
return true;
}else if(Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) { String[] schemaVersionArr = schemaVersion.split("\\.");
return false; String[] versionArr = version.split("\\.");
} int arrLength = Math.min(schemaVersionArr.length, versionArr.length);
} for (int i = 0; i < arrLength; i++) {
if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) {
// If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number return true;
return schemaVersionArr.length > versionArr.length; } else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) {
} return false;
}
/** }
* Gets the current software version number of the system
* @return current software version // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number
*/ return schemaVersionArr.length > versionArr.length;
public static String getSoftVersion() { }
String soft_version;
try { /**
soft_version = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version"))); * Gets the current software version number of the system
soft_version = replaceBlank(soft_version); *
} catch (FileNotFoundException e) { * @return current software version
logger.error(e.getMessage(),e); */
throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); public static String getSoftVersion() {
} String softVersion;
return soft_version; try {
} softVersion = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version")));
softVersion = replaceBlank(softVersion);
/** } catch (FileNotFoundException e) {
* Strips the string of space carriage returns and tabs logger.error(e.getMessage(), e);
* @param str string throw new RuntimeException("Failed to get the product version description file. The file could not be found", e);
* @return string removed blank }
*/ return softVersion;
public static String replaceBlank(String str) { }
String dest = "";
if (str!=null) { /**
* Strips the string of space carriage returns and tabs
Matcher m = p.matcher(str); *
dest = m.replaceAll(""); * @param str string
} * @return string removed blank
return dest; */
} public static String replaceBlank(String str) {
String dest = "";
if (str != null) {
Matcher m = p.matcher(str);
dest = m.replaceAll("");
}
return dest;
}
} }

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java

@ -14,21 +14,25 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
/** /**
* sensitive log Util * sensitive log Util
*/ */
public class SensitiveLogUtils { public class SensitiveLogUtils {
private SensitiveLogUtils() {
throw new UnsupportedOperationException("Construct SensitiveLogUtils");
}
/** /**
* @param dataSourcePwd data source password * @param dataSourcePwd data source password
* @return String * @return String
*/ */
public static String maskDataSourcePwd(String dataSourcePwd){ public static String maskDataSourcePwd(String dataSourcePwd) {
if (StringUtils.isNotEmpty(dataSourcePwd)) { if (StringUtils.isNotEmpty(dataSourcePwd)) {
dataSourcePwd = Constants.PASSWORD_DEFAULT; dataSourcePwd = Constants.PASSWORD_DEFAULT;

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import java.util.Iterator; import java.util.Iterator;
@ -22,7 +23,9 @@ import java.util.stream.StreamSupport;
public class StreamUtils { public class StreamUtils {
private StreamUtils() { } private StreamUtils() {
throw new UnsupportedOperationException("Construct StreamUtils");
}
public static <T> Stream<T> asStream(Iterator<T> sourceIterator) { public static <T> Stream<T> asStream(Iterator<T> sourceIterator) {
return asStream(sourceIterator, false); return asStream(sourceIterator, false);

12
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -14,11 +14,17 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
public class StringUtils { public class StringUtils {
public static final String EMPTY = ""; public static final String EMPTY = "";
private StringUtils() {
throw new UnsupportedOperationException("Construct StringUtils");
}
public static boolean isEmpty(final CharSequence cs) { public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0; return cs == null || cs.length() == 0;
} }
@ -27,18 +33,18 @@ public class StringUtils {
return !isEmpty(cs); return !isEmpty(cs);
} }
public static boolean isBlank(String s){ public static boolean isBlank(String s) {
if (isEmpty(s)) { if (isEmpty(s)) {
return true; return true;
} }
return s.trim().length() == 0; return s.trim().length() == 0;
} }
public static boolean isNotBlank(String s){ public static boolean isNotBlank(String s) {
return !isBlank(s); return !isBlank(s);
} }
public static String replaceNRTtoUnderline(String src){ public static String replaceNRTtoUnderline(String src) {
return src.replaceAll("[\n|\r|\t]", "_"); return src.replaceAll("[\n|\r|\t]", "_");
} }
} }

100
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java

@ -14,13 +14,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.task.datax.DataxParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters;
@ -31,60 +32,65 @@ import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
* task parameters utils * task parameters utils
*/ */
public class TaskParametersUtils { public class TaskParametersUtils {
private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class); private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class);
private TaskParametersUtils() {
throw new UnsupportedOperationException("Construct TaskParametersUtils");
}
/** /**
* get task parameters * get task parameters
* @param taskType task type *
* @param parameter parameter * @param taskType task type
* @return task parameters * @param parameter parameter
*/ * @return task parameters
public static AbstractParameters getParameters(String taskType, String parameter) { */
try { public static AbstractParameters getParameters(String taskType, String parameter) {
switch (EnumUtils.getEnum(TaskType.class,taskType)) { try {
case SUB_PROCESS: switch (EnumUtils.getEnum(TaskType.class, taskType)) {
return JSONUtils.parseObject(parameter, SubProcessParameters.class); case SUB_PROCESS:
case WATERDROP: return JSONUtils.parseObject(parameter, SubProcessParameters.class);
return JSONUtils.parseObject(parameter, ShellParameters.class); case WATERDROP:
case SHELL: return JSONUtils.parseObject(parameter, ShellParameters.class);
return JSONUtils.parseObject(parameter, ShellParameters.class); case SHELL:
case PROCEDURE: return JSONUtils.parseObject(parameter, ShellParameters.class);
return JSONUtils.parseObject(parameter, ProcedureParameters.class); case PROCEDURE:
case SQL: return JSONUtils.parseObject(parameter, ProcedureParameters.class);
return JSONUtils.parseObject(parameter, SqlParameters.class); case SQL:
case MR: return JSONUtils.parseObject(parameter, SqlParameters.class);
return JSONUtils.parseObject(parameter, MapreduceParameters.class); case MR:
case SPARK: return JSONUtils.parseObject(parameter, MapreduceParameters.class);
return JSONUtils.parseObject(parameter, SparkParameters.class); case SPARK:
case PYTHON: return JSONUtils.parseObject(parameter, SparkParameters.class);
return JSONUtils.parseObject(parameter, PythonParameters.class); case PYTHON:
case DEPENDENT: return JSONUtils.parseObject(parameter, PythonParameters.class);
return JSONUtils.parseObject(parameter, DependentParameters.class); case DEPENDENT:
case FLINK: return JSONUtils.parseObject(parameter, DependentParameters.class);
return JSONUtils.parseObject(parameter, FlinkParameters.class); case FLINK:
case HTTP: return JSONUtils.parseObject(parameter, FlinkParameters.class);
return JSONUtils.parseObject(parameter, HttpParameters.class); case HTTP:
case DATAX: return JSONUtils.parseObject(parameter, HttpParameters.class);
return JSONUtils.parseObject(parameter, DataxParameters.class); case DATAX:
case CONDITIONS: return JSONUtils.parseObject(parameter, DataxParameters.class);
return JSONUtils.parseObject(parameter, ConditionsParameters.class); case CONDITIONS:
case SQOOP: return JSONUtils.parseObject(parameter, ConditionsParameters.class);
return JSONUtils.parseObject(parameter, SqoopParameters.class); case SQOOP:
default: return JSONUtils.parseObject(parameter, SqoopParameters.class);
return null; default:
} return null;
} catch (Exception e) { }
logger.error(e.getMessage(), e); } catch (Exception e) {
logger.error(e.getMessage(), e);
}
return null;
} }
return null;
}
} }

22
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java

@ -14,19 +14,22 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.dao.mapper; package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Param;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/** /**
* process instance mapper interface * process instance mapper interface
*/ */
@ -201,9 +204,20 @@ public interface ProcessInstanceMapper extends BaseMapper<ProcessInstance> {
* @param endTime * @param endTime
* @return ProcessInstance list * @return ProcessInstance list
*/ */
List<ProcessInstance> queryTopNProcessInstance(@Param("size") int size, List<ProcessInstance> queryTopNProcessInstance(@Param("size") int size,
@Param("startTime") Date startTime, @Param("startTime") Date startTime,
@Param("endTime") Date endTime, @Param("endTime") Date endTime,
@Param("status")ExecutionStatus status); @Param("status")ExecutionStatus status);
/**
* query process instance by processDefinitionId and stateArray
* @param processDefinitionId processDefinitionId
* @param states states array
* @return process instance list
*/
List<ProcessInstance> queryByProcessDefineIdAndStatus(
@Param("processDefinitionId") int processDefinitionId,
@Param("states") int[] states);
} }

12
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml

@ -191,6 +191,16 @@
</if> </if>
order by end_time desc limit 1 order by end_time desc limit 1
</select> </select>
<select id="queryByProcessDefineIdAndStatus"
resultType="org.apache.dolphinscheduler.dao.entity.ProcessInstance">
select *
from t_ds_process_instance
where process_definition_id=#{processDefinitionId}
and state in
<foreach collection="states" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
order by id asc
</select>
</mapper> </mapper>

4
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java

@ -27,7 +27,6 @@ import io.netty.channel.EventLoopGroup;
import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
import org.apache.dolphinscheduler.remote.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
@ -155,11 +154,12 @@ public class NettyRemotingClient {
this.bootstrap this.bootstrap
.group(this.workerGroup) .group(this.workerGroup)
.channel(NioSocketChannel.class) .channel(NettyUtils.getSocketChannelClass())
.option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive())
.option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay())
.option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize())
.option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize())
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis())
.handler(new ChannelInitializer<SocketChannel>() { .handler(new ChannelInitializer<SocketChannel>() {
@Override @Override
public void initChannel(SocketChannel ch) throws Exception { public void initChannel(SocketChannel ch) throws Exception {

36
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java

@ -17,17 +17,6 @@
package org.apache.dolphinscheduler.remote; package org.apache.dolphinscheduler.remote;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
import org.apache.dolphinscheduler.remote.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
@ -37,15 +26,25 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Constants;
import org.apache.dolphinscheduler.remote.utils.NettyUtils; import org.apache.dolphinscheduler.remote.utils.NettyUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
/** /**
* remoting netty server * remoting netty server
*/ */
@ -146,17 +145,17 @@ public class NettyRemotingServer {
if (isStarted.compareAndSet(false, true)) { if (isStarted.compareAndSet(false, true)) {
this.serverBootstrap this.serverBootstrap
.group(this.bossGroup, this.workGroup) .group(this.bossGroup, this.workGroup)
.channel(NioServerSocketChannel.class) .channel(NettyUtils.getServerSocketChannelClass())
.option(ChannelOption.SO_REUSEADDR, true) .option(ChannelOption.SO_REUSEADDR, true)
.option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog())
.childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive())
.childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay())
.childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize())
.childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize())
.childHandler(new ChannelInitializer<NioSocketChannel>() { .childHandler(new ChannelInitializer<SocketChannel>() {
@Override @Override
protected void initChannel(NioSocketChannel ch) throws Exception { protected void initChannel(SocketChannel ch) throws Exception {
initNettyChannel(ch); initNettyChannel(ch);
} }
}); });
@ -182,9 +181,8 @@ public class NettyRemotingServer {
* init netty channel * init netty channel
* *
* @param ch socket channel * @param ch socket channel
* @throws Exception
*/ */
private void initNettyChannel(NioSocketChannel ch) throws Exception { private void initNettyChannel(SocketChannel ch) {
ChannelPipeline pipeline = ch.pipeline(); ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast("encoder", encoder); pipeline.addLast("encoder", encoder);
pipeline.addLast("decoder", new NettyDecoder()); pipeline.addLast("decoder", new NettyDecoder());

23
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java

@ -14,22 +14,23 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.remote.config; package org.apache.dolphinscheduler.remote.config;
import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Constants;
/** /**
* netty client config * netty client config
*/ */
public class NettyClientConfig { public class NettyClientConfig {
/** /**
* worker threadsdefault get machine cpus * worker threadsdefault get machine cpus
*/ */
private int workerThreads = Constants.CPUS; private int workerThreads = Constants.CPUS;
/** /**
* whether tpc delay * whether tpc delay
*/ */
private boolean tcpNoDelay = true; private boolean tcpNoDelay = true;
@ -39,15 +40,20 @@ public class NettyClientConfig {
private boolean soKeepalive = true; private boolean soKeepalive = true;
/** /**
* send buffer size * send buffer size
*/ */
private int sendBufferSize = 65535; private int sendBufferSize = 65535;
/** /**
* receive buffer size * receive buffer size
*/ */
private int receiveBufferSize = 65535; private int receiveBufferSize = 65535;
/**
* connect timeout millis
*/
private int connectTimeoutMillis = 3000;
public int getWorkerThreads() { public int getWorkerThreads() {
return workerThreads; return workerThreads;
} }
@ -88,4 +94,11 @@ public class NettyClientConfig {
this.receiveBufferSize = receiveBufferSize; this.receiveBufferSize = receiveBufferSize;
} }
public int getConnectTimeoutMillis() {
return connectTimeoutMillis;
}
public void setConnectTimeoutMillis(int connectTimeoutMillis) {
this.connectTimeoutMillis = connectTimeoutMillis;
}
} }

9
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.remote.utils; package org.apache.dolphinscheduler.remote.utils;
import java.nio.charset.Charset; import java.nio.charset.Charset;
@ -21,7 +22,7 @@ import java.nio.charset.StandardCharsets;
/** /**
* constant * constant
*/ */
public class Constants { public class Constants {
@ -30,12 +31,12 @@ public class Constants {
public static final String SLASH = "/"; public static final String SLASH = "/";
/** /**
* charset * charset
*/ */
public static final Charset UTF8 = StandardCharsets.UTF_8; public static final Charset UTF8 = StandardCharsets.UTF_8;
/** /**
* cpus * cpus
*/ */
public static final int CPUS = Runtime.getRuntime().availableProcessors(); public static final int CPUS = Runtime.getRuntime().availableProcessors();
@ -45,7 +46,7 @@ public class Constants {
/** /**
* netty epoll enable switch * netty epoll enable switch
*/ */
public static final String NETTY_EPOLL_ENABLE = System.getProperty("netty.epoll.enable"); public static final String NETTY_EPOLL_ENABLE = System.getProperty("netty.epoll.enable", "true");
/** /**
* OS Name * OS Name

20
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java

@ -18,6 +18,12 @@
package org.apache.dolphinscheduler.remote.utils; package org.apache.dolphinscheduler.remote.utils;
import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.Epoll;
import io.netty.channel.epoll.EpollServerSocketChannel;
import io.netty.channel.epoll.EpollSocketChannel;
import io.netty.channel.socket.ServerSocketChannel;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
/** /**
* NettyUtils * NettyUtils
@ -39,4 +45,18 @@ public class NettyUtils {
return Boolean.parseBoolean(enableNettyEpoll); return Boolean.parseBoolean(enableNettyEpoll);
} }
public static Class<? extends ServerSocketChannel> getServerSocketChannelClass() {
if (useEpoll()) {
return EpollServerSocketChannel.class;
}
return NioServerSocketChannel.class;
}
public static Class<? extends SocketChannel> getSocketChannelClass() {
if (useEpoll()) {
return EpollSocketChannel.class;
}
return NioSocketChannel.class;
}
} }

12
dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java

@ -17,20 +17,28 @@
package org.apache.dolphinscheduler.remote; package org.apache.dolphinscheduler.remote;
import static org.apache.dolphinscheduler.remote.utils.Constants.OS_NAME;
import org.apache.dolphinscheduler.remote.utils.NettyUtils; import org.apache.dolphinscheduler.remote.utils.NettyUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import io.netty.channel.epoll.Epoll;
/** /**
* NettyUtilTest * NettyUtilTest
*/ */
public class NettyUtilTest { public class NettyUtilTest {
@Test @Test
public void testUserEpoll() { public void testUserEpoll() {
System.setProperty("netty.epoll.enable", "false"); if (OS_NAME.toLowerCase().contains("linux") && Epoll.isAvailable()) {
Assert.assertFalse(NettyUtils.useEpoll()); Assert.assertTrue(NettyUtils.useEpoll());
} else {
Assert.assertFalse(NettyUtils.useEpoll());
}
} }
} }

2
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java

@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
/** /**
* task fulture * task future
*/ */
public class TaskFuture { public class TaskFuture {

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java

@ -27,6 +27,8 @@ import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.LogUtils;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
@ -123,7 +125,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread {
} }
private void initTaskParameters() { private void initTaskParameters() {
this.taskInstance.setLogPath(getTaskLogPath(taskInstance)); this.taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance));
this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort());
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date()); taskInstance.setStartTime(new Date());

20
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java

@ -14,9 +14,11 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import com.fasterxml.jackson.annotation.JsonFormat; import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DependResult; import org.apache.dolphinscheduler.common.enums.DependResult;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
@ -24,16 +26,22 @@ import org.apache.dolphinscheduler.common.model.DependentTaskModel;
import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.task.dependent.DependentParameters;
import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.DependentUtils;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.utils.LogUtils;
import org.apache.dolphinscheduler.server.utils.DependentExecute; import org.apache.dolphinscheduler.server.utils.DependentExecute;
import org.slf4j.LoggerFactory;
import java.util.*; import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonFormat;
public class DependentTaskExecThread extends MasterBaseTaskExecThread { public class DependentTaskExecThread extends MasterBaseTaskExecThread {
@ -172,7 +180,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread {
} }
private void initTaskParameters() { private void initTaskParameters() {
taskInstance.setLogPath(getTaskLogPath(taskInstance)); taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance));
taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort());
taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(new Date()); taskInstance.setStartTime(new Date());

62
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java

@ -14,17 +14,16 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
@ -36,10 +35,6 @@ import java.util.concurrent.Callable;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.sift.SiftingAppender;
/** /**
* master task exec base class * master task exec base class
*/ */
@ -85,11 +80,13 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
* taskUpdateQueue * taskUpdateQueue
*/ */
private TaskPriorityQueue taskUpdateQueue; private TaskPriorityQueue taskUpdateQueue;
/** /**
* constructor of MasterBaseTaskExecThread * constructor of MasterBaseTaskExecThread
* @param taskInstance task instance *
* @param taskInstance task instance
*/ */
public MasterBaseTaskExecThread(TaskInstance taskInstance){ public MasterBaseTaskExecThread(TaskInstance taskInstance) {
this.processService = SpringApplicationContext.getBean(ProcessService.class); this.processService = SpringApplicationContext.getBean(ProcessService.class);
this.alertDao = SpringApplicationContext.getBean(AlertDao.class); this.alertDao = SpringApplicationContext.getBean(AlertDao.class);
this.cancel = false; this.cancel = false;
@ -100,24 +97,26 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
/** /**
* get task instance * get task instance
*
* @return TaskInstance * @return TaskInstance
*/ */
public TaskInstance getTaskInstance(){ public TaskInstance getTaskInstance() {
return this.taskInstance; return this.taskInstance;
} }
/** /**
* kill master base task exec thread * kill master base task exec thread
*/ */
public void kill(){ public void kill() {
this.cancel = true; this.cancel = true;
} }
/** /**
* submit master base task exec thread * submit master base task exec thread
*
* @return TaskInstance * @return TaskInstance
*/ */
protected TaskInstance submit(){ protected TaskInstance submit() {
Integer commitRetryTimes = masterConfig.getMasterTaskCommitRetryTimes(); Integer commitRetryTimes = masterConfig.getMasterTaskCommitRetryTimes();
Integer commitRetryInterval = masterConfig.getMasterTaskCommitInterval(); Integer commitRetryInterval = masterConfig.getMasterTaskCommitInterval();
@ -156,14 +155,13 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
} }
/** /**
* dispatcht task * dispatcht task
*
* @param taskInstance taskInstance * @param taskInstance taskInstance
* @return whether submit task success * @return whether submit task success
*/ */
public Boolean dispatchTask(TaskInstance taskInstance) { public Boolean dispatchTask(TaskInstance taskInstance) {
try{ try{
if(taskInstance.isConditionsTask() if(taskInstance.isConditionsTask()
|| taskInstance.isDependTask() || taskInstance.isDependTask()
@ -202,7 +200,7 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
/** /**
* buildTaskPriorityInfo * buildTaskPriorityInfo
* *
* @param processInstancePriority processInstancePriority * @param processInstancePriority processInstancePriority
* @param processInstanceId processInstanceId * @param processInstanceId processInstanceId
@ -215,7 +213,7 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
int processInstanceId, int processInstanceId,
int taskInstancePriority, int taskInstancePriority,
int taskInstanceId, int taskInstanceId,
String workerGroup){ String workerGroup) {
return processInstancePriority + return processInstancePriority +
UNDERLINE + UNDERLINE +
processInstanceId + processInstanceId +
@ -229,14 +227,16 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
/** /**
* submit wait complete * submit wait complete
*
* @return true * @return true
*/ */
protected Boolean submitWaitComplete(){ protected Boolean submitWaitComplete() {
return true; return true;
} }
/** /**
* call * call
*
* @return boolean * @return boolean
* @throws Exception exception * @throws Exception exception
*/ */
@ -246,34 +246,4 @@ public class MasterBaseTaskExecThread implements Callable<Boolean> {
return submitWaitComplete(); return submitWaitComplete();
} }
/**
* get task log path
* @return log path
*/
public String getTaskLogPath(TaskInstance task) {
String logPath;
try{
String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory())
.getLogger("ROOT")
.getAppender("TASKLOGFILE"))
.getDiscriminator()).getLogBase();
if (baseLog.startsWith(Constants.SINGLE_SLASH)){
logPath = baseLog + Constants.SINGLE_SLASH +
task.getProcessDefinitionId() + Constants.SINGLE_SLASH +
task.getProcessInstanceId() + Constants.SINGLE_SLASH +
task.getId() + ".log";
}else{
logPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH +
baseLog + Constants.SINGLE_SLASH +
task.getProcessDefinitionId() + Constants.SINGLE_SLASH +
task.getProcessInstanceId() + Constants.SINGLE_SLASH +
task.getId() + ".log";
}
}catch (Exception e){
logger.error("logger", e);
logPath = "";
}
return logPath;
}
} }

76
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import javax.transaction.NotSupportedException;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.sift.SiftingAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.spi.AppenderAttachable;
public class LogUtils {
private LogUtils() throws NotSupportedException {
throw new NotSupportedException();
}
/**
* get task log path
*/
@SuppressWarnings("unchecked")
private static String getTaskLogPath(int processDefinitionId, int processInstanceId, int taskInstanceId) {
// Optional.map will be skipped if null
return Optional.of(LoggerFactory.getILoggerFactory())
.map(e -> (AppenderAttachable<ILoggingEvent>) (e.getLogger("ROOT")))
.map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE")))
.map(e -> ((TaskLogDiscriminator) (e.getDiscriminator())))
.map(TaskLogDiscriminator::getLogBase)
.map(e -> Paths.get(e)
.toAbsolutePath()
.resolve(String.valueOf(processDefinitionId))
.resolve(String.valueOf(processInstanceId))
.resolve(taskInstanceId + ".log"))
.map(Path::toString)
.orElse("");
}
/**
* get task log path by TaskInstance
*/
public static String getTaskLogPath(TaskInstance taskInstance) {
return getTaskLogPath(taskInstance.getProcessDefinitionId(), taskInstance.getProcessInstanceId(), taskInstance.getId());
}
/**
* get task log path by TaskExecutionContext
*/
public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) {
return getTaskLogPath(taskExecutionContext.getProcessId(), taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId());
}
}

30
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.server.worker.processor; package org.apache.dolphinscheduler.server.worker.processor;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.thread.ThreadUtils;
@ -36,7 +34,7 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.utils.LogUtils;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
@ -51,8 +49,6 @@ import org.slf4j.LoggerFactory;
import com.github.rholder.retry.RetryException; import com.github.rholder.retry.RetryException;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.sift.SiftingAppender;
import io.netty.channel.Channel; import io.netty.channel.Channel;
/** /**
@ -154,28 +150,6 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
} }
} }
/**
* get task log path
* @return log path
*/
private String getTaskLogPath(TaskExecutionContext taskExecutionContext) {
String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory())
.getLogger("ROOT")
.getAppender("TASKLOGFILE"))
.getDiscriminator()).getLogBase();
if (baseLog.startsWith(Constants.SINGLE_SLASH)){
return baseLog + Constants.SINGLE_SLASH +
taskExecutionContext.getProcessDefineId() + Constants.SINGLE_SLASH +
taskExecutionContext.getProcessInstanceId() + Constants.SINGLE_SLASH +
taskExecutionContext.getTaskInstanceId() + ".log";
}
return System.getProperty("user.dir") + Constants.SINGLE_SLASH +
baseLog + Constants.SINGLE_SLASH +
taskExecutionContext.getProcessDefineId() + Constants.SINGLE_SLASH +
taskExecutionContext.getProcessInstanceId() + Constants.SINGLE_SLASH +
taskExecutionContext.getTaskInstanceId() + ".log";
}
/** /**
* build ack command * build ack command
* @param taskExecutionContext taskExecutionContext * @param taskExecutionContext taskExecutionContext
@ -185,7 +159,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand(); TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand();
ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
ackCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); ackCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
ackCommand.setLogPath(getTaskLogPath(taskExecutionContext)); ackCommand.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext));
ackCommand.setHost(taskExecutionContext.getHost()); ackCommand.setHost(taskExecutionContext.getHost());
ackCommand.setStartTime(taskExecutionContext.getStartTime()); ackCommand.setStartTime(taskExecutionContext.getStartTime());
if(taskExecutionContext.getTaskType().equals(TaskType.SQL.name()) || taskExecutionContext.getTaskType().equals(TaskType.PROCEDURE.name())){ if(taskExecutionContext.getTaskType().equals(TaskType.SQL.name()) || taskExecutionContext.getTaskType().equals(TaskType.PROCEDURE.name())){

159
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java

@ -0,0 +1,159 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.master;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.runner.SubProcessTaskExecThread;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.context.ApplicationContext;
@RunWith(PowerMockRunner.class)
@PrepareForTest({
Stopper.class,
})
public class SubProcessTaskTest {
/**
* TaskNode.runFlag : task can be run normally
*/
public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL";
private ProcessService processService;
private ProcessInstance processInstance;
@Before
public void before() {
ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class);
SpringApplicationContext springApplicationContext = new SpringApplicationContext();
springApplicationContext.setApplicationContext(applicationContext);
MasterConfig config = new MasterConfig();
Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
config.setMasterTaskCommitRetryTimes(3);
config.setMasterTaskCommitInterval(1000);
PowerMockito.mockStatic(Stopper.class);
PowerMockito.when(Stopper.isRunning()).thenReturn(true);
processService = Mockito.mock(ProcessService.class);
Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
processInstance = getProcessInstance();
Mockito.when(processService
.findProcessInstanceById(processInstance.getId()))
.thenReturn(processInstance);
// for SubProcessTaskExecThread.setTaskInstanceState
Mockito.when(processService
.updateTaskInstance(Mockito.any()))
.thenReturn(true);
// for MasterBaseTaskExecThread.submit
Mockito.when(processService
.submitTask(Mockito.any()))
.thenAnswer(t -> t.getArgument(0));
}
private TaskInstance testBasicInit(ExecutionStatus expectResult) {
TaskInstance taskInstance = getTaskInstance(getTaskNode(), processInstance);
ProcessInstance subProcessInstance = getSubProcessInstance(expectResult);
// for SubProcessTaskExecThread.waitTaskQuit
Mockito.when(processService
.findProcessInstanceById(subProcessInstance.getId()))
.thenReturn(subProcessInstance);
Mockito.when(processService
.findSubProcessInstance(processInstance.getId(), taskInstance.getId()))
.thenReturn(subProcessInstance);
return taskInstance;
}
@Test
public void testBasicSuccess() throws Exception {
TaskInstance taskInstance = testBasicInit(ExecutionStatus.SUCCESS);
SubProcessTaskExecThread taskExecThread = new SubProcessTaskExecThread(taskInstance);
taskExecThread.call();
Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecThread.getTaskInstance().getState());
}
@Test
public void testBasicFailure() throws Exception {
TaskInstance taskInstance = testBasicInit(ExecutionStatus.FAILURE);
SubProcessTaskExecThread taskExecThread = new SubProcessTaskExecThread(taskInstance);
taskExecThread.call();
Assert.assertEquals(ExecutionStatus.FAILURE, taskExecThread.getTaskInstance().getState());
}
private TaskNode getTaskNode() {
TaskNode taskNode = new TaskNode();
taskNode.setId("tasks-10");
taskNode.setName("S");
taskNode.setType(TaskType.SUB_PROCESS.toString());
taskNode.setRunFlag(FLOWNODE_RUN_FLAG_NORMAL);
return taskNode;
}
private ProcessInstance getProcessInstance() {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setId(100);
processInstance.setProcessDefinitionId(1);
processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
return processInstance;
}
private ProcessInstance getSubProcessInstance(ExecutionStatus executionStatus) {
ProcessInstance processInstance = new ProcessInstance();
processInstance.setId(102);
processInstance.setProcessDefinitionId(2);
processInstance.setState(executionStatus);
return processInstance;
}
private TaskInstance getTaskInstance(TaskNode taskNode, ProcessInstance processInstance) {
TaskInstance taskInstance = new TaskInstance();
taskInstance.setId(1000);
taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode));
taskInstance.setName(taskNode.getName());
taskInstance.setTaskType(taskNode.getType());
taskInstance.setProcessInstanceId(processInstance.getId());
taskInstance.setProcessDefinitionId(processInstance.getProcessDefinitionId());
taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
return taskInstance;
}
}

46
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java

@ -14,67 +14,53 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.server.master.dispatch.host; package org.apache.dolphinscheduler.server.master.dispatch.host;
import com.google.common.collect.Sets;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
import org.apache.dolphinscheduler.server.registry.DependencyConfig;
import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager;
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter;
import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils; import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils;
import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry;
import org.apache.dolphinscheduler.server.zk.SpringZKServer;
import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient;
import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
import org.apache.dolphinscheduler.service.zk.ZookeeperConfig;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired; import org.mockito.InjectMocks;
import org.springframework.test.context.ContextConfiguration; import org.mockito.Mock;
import org.springframework.test.context.junit4.SpringRunner; import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
/** /**
* round robin host manager test * round robin host manager test
*/ */
@RunWith(SpringRunner.class) @RunWith(MockitoJUnitRunner.class)
@ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, WorkerRegistry.class, ZookeeperRegistryCenter.class, WorkerConfig.class,
ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class})
public class RoundRobinHostManagerTest { public class RoundRobinHostManagerTest {
@Autowired @Mock
private ZookeeperNodeManager zookeeperNodeManager; private ZookeeperNodeManager zookeeperNodeManager;
@Autowired @InjectMocks
private WorkerRegistry workerRegistry; RoundRobinHostManager roundRobinHostManager;
@Autowired
private WorkerConfig workerConfig;
@Test @Test
public void testSelectWithEmptyResult(){ public void testSelectWithEmptyResult() {
RoundRobinHostManager roundRobinHostManager = new RoundRobinHostManager(); Mockito.when(zookeeperNodeManager.getWorkerGroupNodes("default")).thenReturn(null);
roundRobinHostManager.setZookeeperNodeManager(zookeeperNodeManager);
ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000);
Host emptyHost = roundRobinHostManager.select(context); Host emptyHost = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isEmpty(emptyHost.getAddress())); Assert.assertTrue(StringUtils.isEmpty(emptyHost.getAddress()));
} }
@Test @Test
public void testSelectWithResult(){ public void testSelectWithResult() {
workerRegistry.registry(); Mockito.when(zookeeperNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22:100"));
RoundRobinHostManager roundRobinHostManager = new RoundRobinHostManager();
roundRobinHostManager.setZookeeperNodeManager(zookeeperNodeManager);
ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000);
Host host = roundRobinHostManager.select(context); Host host = roundRobinHostManager.select(context);
Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress())); Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress()));
Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort())); Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22"));
workerRegistry.unRegistry();
} }
} }

7
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java

@ -17,14 +17,15 @@
package org.apache.dolphinscheduler.server.master.runner; package org.apache.dolphinscheduler.server.master.runner;
import java.util.HashSet;
import java.util.Set;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.HashSet;
import java.util.Set;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;

66
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.sift.SiftingAppender;
@RunWith(MockitoJUnitRunner.class)
public class LogUtilsTest {
@Test
public void testGetTaskLogPath() {
TaskInstance taskInstance = new TaskInstance();
taskInstance.setProcessDefinitionId(1);
taskInstance.setProcessInstanceId(100);
taskInstance.setId(1000);
Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT");
Assert.assertNotNull(rootLogger);
SiftingAppender appender = Mockito.mock(SiftingAppender.class);
// it's a trick to mock logger.getAppend("TASKLOGFILE")
Mockito.when(appender.getName()).thenReturn("TASKLOGFILE");
rootLogger.addAppender(appender);
Path logBase = Paths.get("path").resolve("to").resolve("test");
TaskLogDiscriminator taskLogDiscriminator = Mockito.mock(TaskLogDiscriminator.class);
Mockito.when(taskLogDiscriminator.getLogBase()).thenReturn(logBase.toString());
Mockito.when(appender.getDiscriminator()).thenReturn(taskLogDiscriminator);
Path logPath = Paths.get(".").toAbsolutePath().getParent()
.resolve(logBase)
.resolve("1").resolve("100").resolve("1000.log");
Assert.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskInstance));
}
}

3
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue

@ -790,7 +790,8 @@
props: { props: {
processDefinition: { processDefinition: {
id: self.urlParam.id, id: self.urlParam.id,
version: self.$store.state.dag.version version: self.$store.state.dag.version,
state: self.releaseState
}, },
processDefinitionVersions: processDefinitionVersions, processDefinitionVersions: processDefinitionVersions,
total: total, total: total,

2
dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/udp/udp.vue

@ -82,7 +82,7 @@
</div> </div>
</template> </template>
<x-button type="text" @click="close()"> {{$t('Cancel')}} </x-button> <x-button type="text" @click="close()"> {{$t('Cancel')}} </x-button>
<x-button type="primary" shape="circle" @click="ok()">{{$t('Add')}}</x-button> <x-button type="primary" shape="circle" :disabled="isDetails" @click="ok()">{{$t('Add')}}</x-button>
</div> </div>
</div> </div>
</div> </div>

85
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue

@ -139,6 +139,8 @@
</x-poptip> </x-poptip>
<template v-if="strSelectIds !== ''"> <template v-if="strSelectIds !== ''">
<x-button size="xsmall" style="position: absolute; bottom: -48px; left: 80px;" @click="_batchExport(item)" >{{$t('Export')}}</x-button> <x-button size="xsmall" style="position: absolute; bottom: -48px; left: 80px;" @click="_batchExport(item)" >{{$t('Export')}}</x-button>
<x-button size="xsmall" style="position: absolute; bottom: -48px; left: 140px;" @click="_batchCopy(item)" >{{$t('Batch copy')}}</x-button>
<x-button size="xsmall" style="position: absolute; bottom: -48px; left: 225px;" @click="_batchMove(item)" >{{$t('Batch move')}}</x-button>
</template> </template>
</div> </div>
@ -147,6 +149,7 @@
import _ from 'lodash' import _ from 'lodash'
import mStart from './start' import mStart from './start'
import mTiming from './timing' import mTiming from './timing'
import mRelatedItems from './relatedItems'
import { mapActions } from 'vuex' import { mapActions } from 'vuex'
import { publishStatus } from '@/conf/home/pages/dag/_source/config' import { publishStatus } from '@/conf/home/pages/dag/_source/config'
import mVersions from './versions' import mVersions from './versions'
@ -166,7 +169,7 @@
pageSize: Number pageSize: Number
}, },
methods: { methods: {
...mapActions('dag', ['editProcessState', 'getStartCheck', 'getReceiver', 'deleteDefinition', 'batchDeleteDefinition', 'exportDefinition', 'getProcessDefinitionVersionsPage', 'copyProcess', 'switchProcessDefinitionVersion', 'deleteProcessDefinitionVersion']), ...mapActions('dag', ['editProcessState', 'getStartCheck', 'getReceiver', 'deleteDefinition', 'batchDeleteDefinition', 'exportDefinition', 'getProcessDefinitionVersionsPage', 'copyProcess', 'switchProcessDefinitionVersion', 'deleteProcessDefinitionVersion', 'moveProcess']),
...mapActions('security', ['getWorkerGroupsAll']), ...mapActions('security', ['getWorkerGroupsAll']),
_rtPublishStatus (code) { _rtPublishStatus (code) {
return _.filter(publishStatus, v => v.code === code)[0].desc return _.filter(publishStatus, v => v.code === code)[0].desc
@ -317,8 +320,27 @@
*/ */
_copyProcess (item) { _copyProcess (item) {
this.copyProcess({ this.copyProcess({
processId: item.id processDefinitionIds: item.id,
targetProjectId: item.projectId
}).then(res => {
this.strSelectIds = ''
this.$message.success(res.msg)
$('body').find('.tooltip.fade.top.in').remove()
this._onUpdate()
}).catch(e => {
this.$message.error(e.msg || '')
})
},
/**
* move
*/
_moveProcess (item) {
this.moveProcess({
processDefinitionIds: item.id,
targetProjectId: item.projectId
}).then(res => { }).then(res => {
this.strSelectIds = ''
this.$message.success(res.msg) this.$message.success(res.msg)
$('body').find('.tooltip.fade.top.in').remove() $('body').find('.tooltip.fade.top.in').remove()
this._onUpdate() this._onUpdate()
@ -469,7 +491,64 @@
this.$message.error(e.msg) this.$message.error(e.msg)
}) })
}, },
/**
* Batch Copy
*/
_batchCopy () {
let self = this
let modal = this.$modal.dialog({
closable: false,
showMask: true,
escClose: true,
className: 'v-modal-custom',
transitionName: 'opacityp',
render (h) {
return h(mRelatedItems, {
on: {
onBatchCopy (item) {
self._copyProcess({id: self.strSelectIds,projectId: item})
modal.remove()
},
close () {
modal.remove()
}
},
props: {
tmp: false
}
})
}
})
},
/**
* _batchMove
*/
_batchMove() {
let self = this
let modal = this.$modal.dialog({
closable: false,
showMask: true,
escClose: true,
className: 'v-modal-custom',
transitionName: 'opacityp',
render (h) {
return h(mRelatedItems, {
on: {
onBatchMove (item) {
self._moveProcess({id: self.strSelectIds,projectId: item})
modal.remove()
},
close () {
modal.remove()
}
},
props: {
tmp: true
}
})
}
})
},
/** /**
* Edit state * Edit state
*/ */

94
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/relatedItems.vue

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<template>
<m-popup
ref="popup"
:ok-text="$t('Confirm')"
:nameText="$t('Related items')"
@ok="_ok">
<template slot="content">
<div class="create-tenement-model">
<m-list-box-f>
<template slot="name"><strong>*</strong>{{$t('Project Name')}}</template>
<template slot="content">
<x-select v-model="itemId">
<x-option
v-for="item in itemList"
:key="item.id"
:value="item.id"
:label="item.name">
</x-option>
</x-select>
</template>
</m-list-box-f>
</div>
</template>
</m-popup>
</template>
<script>
import _ from 'lodash'
import i18n from '@/module/i18n'
import store from '@/conf/home/store'
import mPopup from '@/module/components/popup/popup'
import mListBoxF from '@/module/components/listBoxF/listBoxF'
export default {
name: 'create-tenement',
data () {
return {
store,
itemList: [],
itemId: ''
}
},
props: {
tmp: Boolean
},
methods: {
_ok () {
if(this._verification()) {
if(this.tmp) {
this.$emit('onBatchMove',this.itemId)
} else {
this.$emit('onBatchCopy',this.itemId)
}
}
},
_verification() {
if(!this.itemId) {
this.$message.warning(`${i18n.$t('Project name is required')}`)
return false
}
return true
}
},
watch: {
},
created () {
this.store.dispatch('dag/getAllItems', {}).then(res => {
if(res.data.length> 0) {
this.itemList = res.data
}
})
},
mounted () {
},
components: { mPopup, mListBoxF }
}
</script>

59
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue

@ -25,13 +25,10 @@
<table class="fixed"> <table class="fixed">
<caption><!-- placeHolder --></caption> <caption><!-- placeHolder --></caption>
<tr> <tr>
<th scope="col"> <th scope="col" style="min-width: 40px;text-align: left">
<span>#</span> <span>{{$t('Version')}}</span>
</th> </th>
<th scope="col" style="min-width: 40px"> <th scope="col" style="min-width: 30px">
<span>Version</span>
</th>
<th scope="col" style="min-width: 200px;max-width: 300px;">
<span>{{$t('Description')}}</span> <span>{{$t('Description')}}</span>
</th> </th>
<th scope="col" style="min-width: 50px"> <th scope="col" style="min-width: 50px">
@ -42,9 +39,6 @@
</th> </th>
</tr> </tr>
<tr v-for="(item, $index) in processDefinitionVersions" :key="item.id"> <tr v-for="(item, $index) in processDefinitionVersions" :key="item.id">
<td>
<span>-</span>
</td>
<td> <td>
<span v-if="item.version"> <span v-if="item.version">
<span v-if="item.version === processDefinition.version" style="color: green"><strong>{{item.version}} {{$t('Current Version')}}</strong></span> <span v-if="item.version === processDefinition.version" style="color: green"><strong>{{item.version}} {{$t('Current Version')}}</strong></span>
@ -52,7 +46,7 @@
</span> </span>
<span v-else>-</span> <span v-else>-</span>
</td> </td>
<td> <td style="word-break:break-all;">
<span v-if="item.description">{{item.description}}</span> <span v-if="item.description">{{item.description}}</span>
<span v-else>-</span> <span v-else>-</span>
</td> </td>
@ -64,11 +58,15 @@
<x-poptip <x-poptip
:ref="'poptip-switch-version-' + $index" :ref="'poptip-switch-version-' + $index"
placement="top-end" placement="top-end"
width="90"> width="260">
<p>{{$t('Confirm Switch To This Version?')}}</p> <p>{{$t('Confirm Switch To This Version?')}}</p>
<div style="text-align: right; margin: 0;padding-top: 4px;"> <div style="text-align: right; margin: 0;padding-top: 4px;">
<x-button type="text" size="xsmall" shape="circle" @click="_closeSwitchVersion($index)">{{$t('Cancel')}}</x-button> <x-button type="text" size="xsmall" shape="circle" @click="_closeSwitchVersion($index)">
<x-button type="primary" size="xsmall" shape="circle" @click="_mVersionSwitchProcessDefinitionVersion(item)">{{$t('Confirm')}}</x-button> {{$t('Cancel')}}
</x-button>
<x-button type="primary" size="xsmall" shape="circle"
@click="_mVersionSwitchProcessDefinitionVersion(item)">{{$t('Confirm')}}
</x-button>
</div> </div>
<template slot="reference"> <template slot="reference">
<x-button <x-button
@ -76,7 +74,7 @@
type="primary" type="primary"
shape="circle" shape="circle"
size="xsmall" size="xsmall"
:disabled="item.version === processDefinition.version" :disabled="item.version === processDefinition.version || 'ONLINE' === processDefinition.state"
data-toggle="tooltip" data-toggle="tooltip"
:title="$t('Switch To This Version')"> :title="$t('Switch To This Version')">
</x-button> </x-button>
@ -88,8 +86,11 @@
width="90"> width="90">
<p>{{$t('Delete?')}}</p> <p>{{$t('Delete?')}}</p>
<div style="text-align: right; margin: 0;padding-top: 4px;"> <div style="text-align: right; margin: 0;padding-top: 4px;">
<x-button type="text" size="xsmall" shape="circle" @click="_closeDelete($index)">{{$t('Cancel')}}</x-button> <x-button type="text" size="xsmall" shape="circle" @click="_closeDelete($index)">{{$t('Cancel')}}
<x-button type="primary" size="xsmall" shape="circle" @click="_mVersionDeleteProcessDefinitionVersion(item,$index)">{{$t('Confirm')}}</x-button> </x-button>
<x-button type="primary" size="xsmall" shape="circle"
@click="_mVersionDeleteProcessDefinitionVersion(item,$index)">{{$t('Confirm')}}
</x-button>
</div> </div>
<template slot="reference"> <template slot="reference">
<x-button <x-button
@ -97,7 +98,7 @@
type="error" type="error"
shape="circle" shape="circle"
size="xsmall" size="xsmall"
:disabled="item.version === processDefinition.version" :disabled="item.version === processDefinition.version || 'ONLINE' === processDefinition.state"
data-toggle="tooltip" data-toggle="tooltip"
:title="$t('delete')"> :title="$t('delete')">
</x-button> </x-button>
@ -114,8 +115,9 @@
<div v-if="processDefinitionVersions.length > 0"> <div v-if="processDefinitionVersions.length > 0">
<div class="bottom-box"> <div class="bottom-box">
<x-button type="text" @click="_close()"> {{$t('Cancel')}} </x-button> <x-button type="text" @click="_close()"> {{$t('Cancel')}}</x-button>
<x-page :current="pageNo" :total="total" @on-change="_mVersionGetProcessDefinitionVersionsPage" small><!----></x-page> <x-page :current="pageNo" :total="total" @on-change="_mVersionGetProcessDefinitionVersionsPage" small>
<!----></x-page>
</div> </div>
</div> </div>
@ -187,18 +189,14 @@
* Close the switch version layer * Close the switch version layer
*/ */
_closeSwitchVersion (i) { _closeSwitchVersion (i) {
if (i > 0) { this.$refs[`poptip-switch-version-${i}`][0].doClose()
this.$refs[`poptip-switch-version-${i}`][0].doClose()
}
}, },
/** /**
* Close the delete layer * Close the delete layer
*/ */
_closeDelete (i) { _closeDelete (i) {
if (i > 0) { this.$refs[`poptip-delete-${i}`][0].doClose()
this.$refs[`poptip-delete-${i}`][0].doClose()
}
}, },
/** /**
@ -211,8 +209,10 @@
}) })
} }
}, },
created () {}, created () {
mounted () {}, },
mounted () {
},
components: { mNoData } components: { mNoData }
} }
</script> </script>
@ -221,10 +221,12 @@
.container { .container {
width: 500px; width: 500px;
position: relative; position: relative;
.title-box { .title-box {
height: 61px; height: 61px;
border-bottom: 1px solid #DCDEDC; border-bottom: 1px solid #DCDEDC;
position: relative; position: relative;
.name { .name {
position: absolute; position: absolute;
left: 24px; left: 24px;
@ -232,6 +234,7 @@
font-size: 16px; font-size: 16px;
} }
} }
.bottom-box { .bottom-box {
position: absolute; position: absolute;
bottom: 0; bottom: 0;
@ -242,10 +245,12 @@
line-height: 60px; line-height: 60px;
border-top: 1px solid #DCDEDC; border-top: 1px solid #DCDEDC;
background: #fff; background: #fff;
.ans-page { .ans-page {
display: inline-block; display: inline-block;
} }
} }
.table-box { .table-box {
overflow-y: scroll; overflow-y: scroll;
height: calc(100vh - 61px); height: calc(100vh - 61px);

32
dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js

@ -196,7 +196,8 @@ export default {
copyProcess ({ state }, payload) { copyProcess ({ state }, payload) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
io.post(`projects/${state.projectName}/process/copy`, { io.post(`projects/${state.projectName}/process/copy`, {
processId: payload.processId processDefinitionIds: payload.processDefinitionIds,
targetProjectId: payload.targetProjectId
}, res => { }, res => {
resolve(res) resolve(res)
}).catch(e => { }).catch(e => {
@ -205,6 +206,35 @@ export default {
}) })
}, },
/**
* Get process definition DAG diagram details
*/
moveProcess ({ state }, payload) {
return new Promise((resolve, reject) => {
io.post(`projects/${state.projectName}/process/move`, {
processDefinitionIds: payload.processDefinitionIds,
targetProjectId: payload.targetProjectId
}, res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
/**
* Get all the items created by the logged in user
*/
getAllItems ({ state }, payload) {
return new Promise((resolve, reject) => {
io.get(`projects/login-user-created-project`, {}, res => {
resolve(res)
}).catch(e => {
reject(e)
})
})
},
/** /**
* Get the process instance DAG diagram details * Get the process instance DAG diagram details
*/ */

6
dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@ -638,5 +638,9 @@ export default {
'Connection name': 'Connection name', 'Connection name': 'Connection name',
'Current connection settings': 'Current connection settings', 'Current connection settings': 'Current connection settings',
'Please save the DAG before formatting': 'Please save the DAG before formatting', 'Please save the DAG before formatting': 'Please save the DAG before formatting',
'Batch copy': 'Batch copy' 'Batch copy': 'Batch copy',
'Related items': 'Related items',
'Project name is required': 'Project name is required',
'Batch move': 'Batch move',
Version: 'Version'
} }

6
dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@ -638,5 +638,9 @@ export default {
'Connection name': '连线名', 'Connection name': '连线名',
'Current connection settings': '当前连线设置', 'Current connection settings': '当前连线设置',
'Please save the DAG before formatting': '格式化前请先保存DAG', 'Please save the DAG before formatting': '格式化前请先保存DAG',
'Batch copy': '批量复制' 'Batch copy': '批量复制',
'Related items': '关联项目',
'Project name is required': '项目名称必填',
'Batch move': '批量移动',
Version: '版本'
} }

4
install.sh

@ -98,7 +98,3 @@ sh ${workDir}/script/remove-zk-node.sh $zkRoot
# 6.startup # 6.startup
echo "6.startup" echo "6.startup"
sh ${workDir}/script/start-all.sh sh ${workDir}/script/start-all.sh
# 7.query status
echo "7.query status"
sh ${workDir}/script/status-all.sh

3
pom.xml

@ -821,16 +821,19 @@
<include>**/server/master/dispatch/host/assign/RandomSelectorTest.java</include> <include>**/server/master/dispatch/host/assign/RandomSelectorTest.java</include>
<include>**/server/master/dispatch/host/assign/RoundRobinSelectorTest.java</include> <include>**/server/master/dispatch/host/assign/RoundRobinSelectorTest.java</include>
<include>**/server/master/register/MasterRegistryTest.java</include> <include>**/server/master/register/MasterRegistryTest.java</include>
<include>**/server/master/dispatch/host/assign/RoundRobinHostManagerTest.java</include>
<include>**/server/master/AlertManagerTest.java</include> <include>**/server/master/AlertManagerTest.java</include>
<include>**/server/master/MasterCommandTest.java</include> <include>**/server/master/MasterCommandTest.java</include>
<include>**/server/master/DependentTaskTest.java</include> <include>**/server/master/DependentTaskTest.java</include>
<include>**/server/master/ConditionsTaskTest.java</include> <include>**/server/master/ConditionsTaskTest.java</include>
<include>**/server/master/MasterExecThreadTest.java</include> <include>**/server/master/MasterExecThreadTest.java</include>
<include>**/server/master/ParamsTest.java</include> <include>**/server/master/ParamsTest.java</include>
<include>**/server/master/SubProcessTaskTest.java</include>
<include>**/server/register/ZookeeperNodeManagerTest.java</include> <include>**/server/register/ZookeeperNodeManagerTest.java</include>
<include>**/server/utils/DataxUtilsTest.java</include> <include>**/server/utils/DataxUtilsTest.java</include>
<include>**/server/utils/ExecutionContextTestUtils.java</include> <include>**/server/utils/ExecutionContextTestUtils.java</include>
<!--<include>**/server/utils/FlinkArgsUtilsTest.java</include>--> <!--<include>**/server/utils/FlinkArgsUtilsTest.java</include>-->
<include>**/server/utils/LogUtilsTest.java</include>
<include>**/server/utils/ParamUtilsTest.java</include> <include>**/server/utils/ParamUtilsTest.java</include>
<include>**/server/utils/ProcessUtilsTest.java</include> <include>**/server/utils/ProcessUtilsTest.java</include>
<include>**/server/utils/SparkArgsUtilsTest.java</include> <include>**/server/utils/SparkArgsUtilsTest.java</include>

19
script/dolphinscheduler-daemon.sh

@ -16,7 +16,7 @@
# limitations under the License. # limitations under the License.
# #
usage="Usage: dolphinscheduler-daemon.sh (start|stop) <command> " usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) <command> "
# if no args specified, show usage # if no args specified, show usage
if [ $# -le 1 ]; then if [ $# -le 1 ]; then
@ -29,7 +29,6 @@ shift
command=$1 command=$1
shift shift
echo "Begin $startStop $command......"
BIN_DIR=`dirname $0` BIN_DIR=`dirname $0`
BIN_DIR=`cd "$BIN_DIR"; pwd` BIN_DIR=`cd "$BIN_DIR"; pwd`
@ -122,11 +121,23 @@ case $startStop in
fi fi
;; ;;
(status)
# more details about the status can be added later
serverCount=`ps -ef |grep "$CLASS" |grep -v "grep" |wc -l`
state="STOP"
# font color - red
state="[ \033[1;31m $state \033[0m ]"
if [[ $serverCount -gt 0 ]];then
state="RUNNING"
# font color - green
state="[ \033[1;32m $state \033[0m ]"
fi
echo -e "$command $state"
;;
(*) (*)
echo $usage echo $usage
exit 1 exit 1
;; ;;
esac esac
echo "End $startStop $command."

4
script/start-all.sh

@ -55,3 +55,7 @@ do
echo "$apiServer worker server is starting" echo "$apiServer worker server is starting"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;"
done done
# query server status
echo "query server status"
cd $installPath/; sh bin/status-all.sh

59
script/status-all.sh

@ -20,22 +20,59 @@ workDir=`dirname $0`
workDir=`cd ${workDir};pwd` workDir=`cd ${workDir};pwd`
source $workDir/../conf/config/install_config.conf source $workDir/../conf/config/install_config.conf
# install_config.conf info
echo -e '\n' echo -e '\n'
echo "====================== dolphinscheduler install config=============================" echo "====================== dolphinscheduler server config ============================="
echo -e "1.dolphinscheduler server node install hosts:[ \033[1;32m ${ips} \033[0m ]" echo -e "1.dolphinscheduler server node config hosts:[ \033[1;32m ${ips} \033[0m ]"
echo -e "2.master server node install hosts:[ \033[1;32m ${masters} \033[0m ]" echo -e "2.master server node config hosts:[ \033[1;32m ${masters} \033[0m ]"
echo -e "3.worker server node install hosts:[ \033[1;32m ${workers} \033[0m ]" echo -e "3.worker server node config hosts:[ \033[1;32m ${workers} \033[0m ]"
echo -e "4.alert server node install hosts:[ \033[1;32m ${alertServer} \033[0m ]" echo -e "4.alert server node config hosts:[ \033[1;32m ${alertServer} \033[0m ]"
echo -e "5.api server node install hosts:[ \033[1;32m ${apiServers} \033[0m ]" echo -e "5.api server node config hosts:[ \033[1;32m ${apiServers} \033[0m ]"
# all server check state
echo -e '\n' echo -e '\n'
echo "====================== dolphinscheduler server status ============================="
firstColumn="node server state"
echo $firstColumn
echo -e '\n'
declare -A workersGroupMap=()
workersGroup=(${workers//,/ })
for workerGroup in ${workersGroup[@]}
do
worker=`echo $workerGroup|awk -F':' '{print $1}'`
groupName=`echo $workerGroup|awk -F':' '{print $2}'`
workersGroupMap+=([$worker]=$groupName)
done
ipsHost=(${ips//,/ }) StateRunning="Running"
for ip in ${ipsHost[@]} # 1.master server check state
mastersHost=(${masters//,/ })
for master in ${mastersHost[@]}
do do
echo -e "====================== [ \033[1;32m ${ip} \033[0m ] node all servers ==========================" masterState=`ssh -p $sshPort $master "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status master-server;"`
ssh -p $sshPort $ip "jps" echo "$master $masterState"
echo -e '\n'
done done
# 2.worker server and logger-server check state
for worker in ${!workersGroupMap[*]}
do
workerState=`ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status worker-server;"`
echo "$worker $workerState"
masterState=`ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status logger-server;"`
echo "$worker $masterState"
done
# 3.alter server check state
alertState=`ssh -p $sshPort $alertServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status alert-server;"`
echo "$alertServer $alertState"
# 4.api server check state
apiServersHost=(${apiServers//,/ })
for apiServer in ${apiServersHost[@]}
do
apiState=`ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"`
echo "$apiServer $apiState"
done

4
script/stop-all.sh

@ -55,3 +55,7 @@ do
echo "$apiServer worker server is stopping" echo "$apiServer worker server is stopping"
ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;"
done done
# query server status
echo "query server status"
cd $installPath/; sh bin/status-all.sh

Loading…
Cancel
Save