Browse Source

Merge remote-tracking branch 'upstream/dev' into json_split

# Conflicts:
#	dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
#	dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
#	dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java
#	dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
#	dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
pull/3/MERGE
baoliang 3 years ago
parent
commit
0166aebf71
  1. 45
      .github/workflows/issue_robot.yml
  2. 3
      README.md
  3. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
  4. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
  5. 8
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
  6. 4
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
  7. 17
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
  8. 1
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
  9. 7
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
  10. 12
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java
  11. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
  12. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
  13. 2
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
  14. 13
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
  15. 4
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
  16. 21
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java
  17. 58
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  18. 165
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java
  19. 5
      dolphinscheduler-common/src/main/resources/common.properties
  20. 47
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~HEAD
  21. 47
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~dev
  22. 13
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java
  23. 10
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java
  24. 7
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java
  25. 6
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml
  26. 6
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
  27. 28
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
  28. 11
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
  29. 7
      dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java
  30. 14
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
  31. 18
      dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue
  32. 1
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue
  33. 1
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/_source/list.vue
  34. 2
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue
  35. 1
      dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue
  36. 2
      dolphinscheduler-ui/src/js/module/components/fileUpdate/fileChildUpdate.vue
  37. 2
      dolphinscheduler-ui/src/js/module/components/fileUpdate/resourceChildUpdate.vue

45
.github/workflows/issue_robot.yml

@ -0,0 +1,45 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: issue-robot
on:
issues:
types: [opened]
jobs:
comment:
runs-on: ubuntu-latest
steps:
- uses: ben-z/actions-comment-on-issue@1.0.2
with:
message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order to let us know your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email,Mail subscription steps reference https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html ,then send questions to dev@dolphinscheduler.apache.org."
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
autoTranslate:
runs-on: ubuntu-latest
steps:
- uses: actions-cool/translation-helper@v1.1.1
automate-issues-labels:
runs-on: ubuntu-latest
steps:
- name: initial labeling
uses: andymckay/labeler@master
with:
add-labels: "Waiting for reply"
ignore-if-labeled: true

3
README.md

@ -6,6 +6,9 @@ Dolphin Scheduler Official Website
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
[![Twitter Follow](https://img.shields.io/twitter/follow/dolphinschedule.svg?style=social&label=Follow)](https://twitter.com/dolphinschedule)
[![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw)
[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler)

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java

@ -267,6 +267,7 @@ public enum Status {
DELETE_PROCESS_TASK_RELATION_ERROR(50032, "delete process task relation error", "删除工作流任务关系错误"),
PROCESS_TASK_RELATION_NOT_EXIST(50033, "process task relation {0} does not exist", "工作流任务关系[{0}]不存在"),
PROCESS_TASK_RELATION_EXIST(50034, "process task relation is already exist, processCode:[{0}]", "工作流任务关系已存在, processCode:[{0}]"),
PROCESS_DAG_IS_EMPTY(50035, "process dag can not be empty", "工作流dag不能为空"),
HDFS_NOT_STARTUP(60001, "hdfs not startup", "hdfs未启用"),
/**

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java

@ -488,7 +488,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
separator = ";";
}
parameterMap.put(TYPE, connectType);
parameterMap.put(Constants.ADDRESS, address);
parameterMap.put(Constants.DATABASE, database);
parameterMap.put(Constants.JDBC_URL, jdbcUrl);

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java

@ -24,7 +24,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODE_
import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
import static org.apache.dolphinscheduler.common.Constants.MAX_TASK_TIMEOUT;
import org.apache.commons.collections.MapUtils;
import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.ExecutorService;
@ -57,6 +56,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.commons.collections.MapUtils;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -267,7 +268,10 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
Map<String, Object> commandMap = JSONUtils.toMap(processInstance.getCommandParam(), String.class, Object.class);
String startParams = null;
if (MapUtils.isNotEmpty(commandMap) && executeType == ExecuteType.REPEAT_RUNNING) {
startParams = (commandMap.get(Constants.CMD_PARAM_START_PARAMS)).toString();
Object startParamsJson = commandMap.get(Constants.CMD_PARAM_START_PARAMS);
if (startParamsJson != null) {
startParams = startParamsJson.toString();
}
}
switch (executeType) {

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java

@ -1093,9 +1093,9 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
// Check whether the task node is normal
List<TaskNode> taskNodes = processData.getTasks();
if (taskNodes == null) {
if (CollectionUtils.isEmpty(taskNodes)) {
logger.error("process node info is empty");
putMsg(result, Status.DATA_IS_NULL, processDefinitionJson);
putMsg(result, Status.PROCESS_DAG_IS_EMPTY);
return result;
}

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java

@ -309,7 +309,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
@Override
public Map<String, Object> queryUnauthorizedProject(User loginUser, Integer userId) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser, result)) {
if (loginUser.getId() != userId && isNotAdmin(loginUser, result)) {
return result;
}
/**
@ -361,7 +361,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
public Map<String, Object> queryAuthorizedProject(User loginUser, Integer userId) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser, result)) {
if (loginUser.getId() != userId && isNotAdmin(loginUser, result)) {
return result;
}
@ -382,10 +382,6 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
public Map<String, Object> queryProjectCreatedByUser(User loginUser) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser, result)) {
return result;
}
List<Project> projects = projectMapper.queryProjectCreatedByUser(loginUser.getId());
result.put(Constants.DATA_LIST, projects);
putMsg(result, Status.SUCCESS);
@ -455,19 +451,14 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic
}
/**
* query all project list that have one or more process definitions.
* query all project list
*
* @return project list
*/
@Override
public Map<String, Object> queryAllProjectList() {
Map<String, Object> result = new HashMap<>();
List<Project> projects = new ArrayList<>();
List<Integer> projectIds = processDefinitionMapper.listProjectIds();
if (CollectionUtils.isNotEmpty(projectIds)) {
projects = projectMapper.selectBatchIds(projectIds);
}
List<Project> projects = projectMapper.queryAllProject();
result.put(Constants.DATA_LIST, projects);
putMsg(result, Status.SUCCESS);

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@ -367,7 +367,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
resource.setDescription(desc);
resource.setUpdateTime(now);
if (file != null) {
resource.setFileName(file.getOriginalFilename());
resource.setSize(file.getSize());
}

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java

@ -90,7 +90,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
return result;
}
if (RegexUtils.isNumeric(tenantCode)) {
if (!RegexUtils.isValidLinuxUserName(tenantCode)) {
putMsg(result, Status.CHECK_OS_TENANT_CODE_ERROR);
return result;
}
@ -102,11 +102,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
Tenant tenant = new Tenant();
Date now = new Date();
if (!tenantCode.matches("^[0-9a-zA-Z_.-]{1,}$") || tenantCode.startsWith("-") || tenantCode.startsWith(".")) {
putMsg(result, Status.VERIFY_OS_TENANT_CODE_ERROR);
return result;
}
tenant.setTenantCode(tenantCode);
tenant.setQueueId(queueId);
tenant.setDescription(desc);

12
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java

@ -30,6 +30,8 @@ public class RegexUtils {
*/
private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$";
private static final String LINUX_USERNAME_PATTERN = "[a-z_][a-z\\d_]{0,30}";
private RegexUtils() {
}
@ -45,6 +47,16 @@ public class RegexUtils {
return isNum.matches();
}
/**
* check if the input is a valid linux username
* @param str input
* @return boolean
*/
public static boolean isValidLinuxUserName(String str) {
Pattern pattern = Pattern.compile(LINUX_USERNAME_PATTERN);
return pattern.matcher(str).matches();
}
public static String escapeNRT(String str) {
// Logging should not be vulnerable to injection attacks: Replace pattern-breaking characters
if (str != null && !str.isEmpty()) {

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java

@ -45,7 +45,7 @@ public class TenantControllerTest extends AbstractControllerTest{
@Test
public void testCreateTenant() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("tenantCode","tenantCode");
paramsMap.add("tenantCode","hayden");
paramsMap.add("queueId","1");
paramsMap.add("description","tenant description");
@ -124,7 +124,7 @@ public class TenantControllerTest extends AbstractControllerTest{
@Test
public void testVerifyTenantCodeExists() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("tenantCode", "tenantCode");
paramsMap.add("tenantCode", "hayden");
MvcResult mvcResult = mockMvc.perform(get("/tenant/verify-tenant-code")
.header(SESSION_ID, sessionId)

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java

@ -281,7 +281,7 @@ public class DataSourceServiceTest {
public void buildParameter() {
String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im"
, "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "", "", "", "");
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\","
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\","
+ "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param);
@ -290,7 +290,7 @@ public class DataSourceServiceTest {
PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test");
param = dataSourceService.buildParameter(DbType.HIVE, "192.168.9.1", "10000", "im"
, "hive/hdfs-mycluster@ESZ.COM", "test", "test", null, "", "/opt/krb5.conf", "test2/hdfs-mycluster@ESZ.COM", "/opt/hdfs.headless.keytab");
expected = "{\"type\":null,\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\","
expected = "{\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\","
+ "\"user\":\"test\",\"password\":\"test\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\","
+ "\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
Assert.assertEquals(expected, param);
@ -303,14 +303,14 @@ public class DataSourceServiceTest {
String other = "{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\"}";
String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, other, "", "", "");
String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
String expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
+ "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param);
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, "", "", "", "");
expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
+ "\"user\":\"test\",\"password\":\"123456\"}";
Assert.assertEquals(expected, param);
}

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java

@ -687,7 +687,7 @@ public class ProcessDefinitionServiceTest {
// task empty
processData.setTasks(null);
Map<String, Object> taskNotEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson);
Assert.assertEquals(Status.DATA_IS_NULL, taskNotEmptyRes.get(Constants.STATUS));
Assert.assertEquals(Status.PROCESS_DAG_IS_EMPTY, taskNotEmptyRes.get(Constants.STATUS));
// task cycle
String processDefinitionJsonCycle = CYCLE_SHELL_JSON;

13
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java

@ -276,6 +276,10 @@ public class ProjectServiceTest {
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
loginUser.setUserType(UserType.GENERAL_USER);
result = projectService.queryAuthorizedProject(loginUser, loginUser.getId());
projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
}
@Test
@ -284,14 +288,10 @@ public class ProjectServiceTest {
User loginUser = getLoginUser();
Mockito.when(projectMapper.queryProjectCreatedByUser(1)).thenReturn(getList());
//USER_NO_OPERATION_PERM
Map<String, Object> result = projectService.queryProjectCreatedByUser(loginUser);
logger.info(result.toString());
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//success
loginUser.setUserType(UserType.ADMIN_USER);
result = projectService.queryProjectCreatedByUser(loginUser);
Map<String, Object> result = projectService.queryProjectCreatedByUser(loginUser);
logger.info(result.toString());
List<Project> projects = (List<Project>) result.get(Constants.DATA_LIST);
Assert.assertTrue(CollectionUtils.isNotEmpty(projects));
@ -322,8 +322,7 @@ public class ProjectServiceTest {
@Test
public void testQueryAllProjectList() {
Mockito.when(processDefinitionMapper.listProjectIds()).thenReturn(getProjectIds());
Mockito.when(projectMapper.selectBatchIds(getProjectIds())).thenReturn(getList());
Mockito.when(projectMapper.queryAllProject()).thenReturn(getList());
Map<String, Object> result = projectService.queryAllProjectList();
logger.info(result.toString());

4
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java

@ -73,7 +73,7 @@ public class TenantServiceTest {
@Mock
private UserMapper userMapper;
private static final String tenantCode = "TenantServiceTest";
private static final String tenantCode = "hayden";
@Test
public void testCreateTenant() {
@ -85,7 +85,7 @@ public class TenantServiceTest {
Map<String, Object> result =
tenantService.createTenant(getLoginUser(), "%!1111", 1, "TenantServiceTest");
logger.info(result.toString());
Assert.assertEquals(Status.VERIFY_OS_TENANT_CODE_ERROR, result.get(Constants.STATUS));
Assert.assertEquals(Status.CHECK_OS_TENANT_CODE_ERROR, result.get(Constants.STATUS));
//check exist
result = tenantService.createTenant(loginUser, tenantCode, 1, "TenantServiceTest");

21
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java

@ -36,6 +36,27 @@ public class RegexUtilsTest {
Assert.assertFalse(numeric2);
}
@Test
public void testIsValidLinuxUserName() {
String name1 = "10000";
Assert.assertFalse(RegexUtils.isValidLinuxUserName(name1));
String name2 = "00hayden";
Assert.assertFalse(RegexUtils.isValidLinuxUserName(name2));
String name3 = "hayde123456789123456789123456789";
Assert.assertFalse(RegexUtils.isValidLinuxUserName(name3));
String name4 = "hayd123456789123456789123456789";
Assert.assertTrue(RegexUtils.isValidLinuxUserName(name4));
String name5 = "h";
Assert.assertTrue(RegexUtils.isValidLinuxUserName(name5));
String name6 = "hayden";
Assert.assertTrue(RegexUtils.isValidLinuxUserName(name6));
}
@Test
public void testEscapeNRT() {
String result1 = RegexUtils.escapeNRT("abc\n");

58
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -205,6 +205,7 @@ public class HadoopUtils implements Closeable {
* if rmHaIds is empty, single resourcemanager enabled
* if rmHaIds not empty: resourcemanager HA enabled
*/
yarnEnabled = true;
String appUrl = StringUtils.isEmpty(rmHaIds) ? appAddress : getAppAddress(appAddress, rmHaIds);
if (StringUtils.isBlank(appUrl)) {
@ -213,7 +214,8 @@ public class HadoopUtils implements Closeable {
if (logger.isDebugEnabled()) {
logger.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId);
}
return String.format(appUrl, applicationId);
String activeResourceManagerPort = String.valueOf(PropertyUtils.getInt(Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088));
return String.format(appUrl, activeResourceManagerPort, applicationId);
}
public String getJobHistoryUrl(String applicationId) {
@ -246,7 +248,7 @@ public class HadoopUtils implements Closeable {
*
* @param hdfsFilePath hdfs file path
* @param skipLineNums skip line numbers
* @param limit read how many lines
* @param limit read how many lines
* @return content of file
* @throws IOException errors
*/
@ -281,10 +283,10 @@ public class HadoopUtils implements Closeable {
/**
* copy files between FileSystems
*
* @param srcPath source hdfs path
* @param dstPath destination hdfs path
* @param srcPath source hdfs path
* @param dstPath destination hdfs path
* @param deleteSource whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param overwrite whether to overwrite an existing file
* @return if success or not
* @throws IOException errors
*/
@ -296,10 +298,10 @@ public class HadoopUtils implements Closeable {
* the src file is on the local disk. Add it to FS at
* the given dst name.
*
* @param srcFile local file
* @param dstHdfsPath destination hdfs path
* @param srcFile local file
* @param dstHdfsPath destination hdfs path
* @param deleteSource whether to delete the src
* @param overwrite whether to overwrite an existing file
* @param overwrite whether to overwrite an existing file
* @return if success or not
* @throws IOException errors
*/
@ -316,9 +318,9 @@ public class HadoopUtils implements Closeable {
* copy hdfs file to local
*
* @param srcHdfsFilePath source hdfs file path
* @param dstFile destination file
* @param deleteSource delete source
* @param overwrite overwrite
* @param dstFile destination file
* @param deleteSource delete source
* @param overwrite overwrite
* @return result of copy hdfs file to local
* @throws IOException errors
*/
@ -347,9 +349,9 @@ public class HadoopUtils implements Closeable {
* delete a file
*
* @param hdfsFilePath the path to delete.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
* @return true if delete is successful else false.
* @throws IOException errors
*/
@ -419,7 +421,9 @@ public class HadoopUtils implements Closeable {
String result = Constants.FAILED;
String applicationUrl = getApplicationUrl(applicationId);
logger.info("applicationUrl={}", applicationUrl);
if (logger.isDebugEnabled()) {
logger.debug("generate yarn application url, applicationUrl={}", applicationUrl);
}
String responseContent = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(applicationUrl) : HttpUtils.get(applicationUrl);
if (responseContent != null) {
@ -432,7 +436,9 @@ public class HadoopUtils implements Closeable {
} else {
//may be in job history
String jobHistoryUrl = getJobHistoryUrl(applicationId);
logger.info("jobHistoryUrl={}", jobHistoryUrl);
if (logger.isDebugEnabled()) {
logger.debug("generate yarn job history application url, jobHistoryUrl={}", jobHistoryUrl);
}
responseContent = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(jobHistoryUrl) : HttpUtils.get(jobHistoryUrl);
if (null != responseContent) {
@ -482,7 +488,7 @@ public class HadoopUtils implements Closeable {
/**
* hdfs resource dir
*
* @param tenantCode tenant code
* @param tenantCode tenant code
* @param resourceType resource type
* @return hdfs resource dir
*/
@ -510,7 +516,7 @@ public class HadoopUtils implements Closeable {
* hdfs user dir
*
* @param tenantCode tenant code
* @param userId user id
* @param userId user id
* @return hdfs resource dir
*/
public static String getHdfsUserDir(String tenantCode, int userId) {
@ -531,8 +537,8 @@ public class HadoopUtils implements Closeable {
* get hdfs file name
*
* @param resourceType resource type
* @param tenantCode tenant code
* @param fileName file name
* @param tenantCode tenant code
* @param fileName file name
* @return hdfs file name
*/
public static String getHdfsFileName(ResourceType resourceType, String tenantCode, String fileName) {
@ -546,7 +552,7 @@ public class HadoopUtils implements Closeable {
* get absolute path and name for resource file on hdfs
*
* @param tenantCode tenant code
* @param fileName file name
* @param fileName file name
* @return get absolute path and name for file on hdfs
*/
public static String getHdfsResourceFileName(String tenantCode, String fileName) {
@ -560,7 +566,7 @@ public class HadoopUtils implements Closeable {
* get absolute path and name for udf file on hdfs
*
* @param tenantCode tenant code
* @param fileName file name
* @param fileName file name
* @return get absolute path and name for udf file on hdfs
*/
public static String getHdfsUdfFileName(String tenantCode, String fileName) {
@ -582,7 +588,7 @@ public class HadoopUtils implements Closeable {
* getAppAddress
*
* @param appAddress app address
* @param rmHa resource manager ha
* @param rmHa resource manager ha
* @return app address
*/
public static String getAppAddress(String appAddress, String rmHa) {
@ -631,9 +637,6 @@ public class HadoopUtils implements Closeable {
/**
* get active resourcemanager
*
* @param rmIds
* @return
*/
public static String getAcitveRMName(String rmIds) {
@ -664,9 +667,6 @@ public class HadoopUtils implements Closeable {
/**
* get ResourceManager state
*
* @param url
* @return
*/
public static String getRMState(String url) {

165
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java

@ -19,8 +19,16 @@ package org.apache.dolphinscheduler.common.utils;
import java.util.Iterator;
import java.util.Collection;
/**
* java.lang.String utils class
*/
public class StringUtils {
/**
* The empty String {@code ""}.
*/
public static final String EMPTY = "";
public static final int INDEX_NOT_FOUND = -1;
@ -29,65 +37,100 @@ public class StringUtils {
throw new UnsupportedOperationException("Construct StringUtils");
}
/**
* <p>Checks if a CharSequence is empty ("") or null.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is empty or null
*/
public static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
/**
* <p>Checks if a CharSequence is not empty ("") and not null.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is not empty and not null
*/
public static boolean isNotEmpty(final CharSequence cs) {
return !isEmpty(cs);
}
public static boolean isBlank(String str) {
/**
* <p>Checks if a CharSequence is empty (""), null or whitespace only.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is null, empty or whitespace only
*/
public static boolean isBlank(final CharSequence cs) {
int strLen;
if (str != null && (strLen = str.length()) != 0) {
for (int i = 0; i < strLen; ++i) {
if (!Character.isWhitespace(str.charAt(i))) {
return false;
}
if (cs == null || (strLen = cs.length()) == 0) {
return true;
}
for (int i = 0; i < strLen; i++) {
if (!Character.isWhitespace(cs.charAt(i))) {
return false;
}
}
return true;
}
public static boolean isNotBlank(String s) {
return !isBlank(s);
/**
* <p>Checks if a CharSequence is not empty (""), not null and not whitespace only.</p>
*
* @param cs the CharSequence to check, may be null
* @return {@code true} if the CharSequence is not empty and not null and not whitespace only
*/
public static boolean isNotBlank(final CharSequence cs) {
return !isBlank(cs);
}
public static String trim(String str) {
return str == null ? null : str.trim();
/**
* <p>Replace all strings matching the regular expression \t \n \r with _</p>
*
* @param src the String , may be null
* @return the string that has been replaced
*/
public static String replaceNRTtoUnderline(String src) {
return isBlank(src) ? src : src.replaceAll("[\n|\r|\t]", "_");
}
public static String join(final Iterable<?> iterable, final String separator) {
if (iterable == null) {
return null;
}
Iterator<?> iterator = iterable.iterator();
if (!iterator.hasNext()) {
return EMPTY;
}
final Object first = iterator.next();
if (!iterator.hasNext()) {
return String.valueOf(first);
}
// two or more elements
final StringBuilder buf = new StringBuilder(256);
while (iterator.hasNext()) {
if (separator != null) {
buf.append(separator);
}
final Object obj = iterator.next();
if (obj != null) {
buf.append(obj);
}
}
return buf.toString();
/**
* <p>Removes control characters (char &lt;= 32) from both
* ends of this String, handling {@code null} by returning
* {@code null}.</p>
*
* @param str the String to be trimmed, may be null
* @return the trimmed string, {@code null} if null String input
*/
public static String trim(final String str) {
return str == null ? null : str.trim();
}
public static String defaultIfBlank(String str, String defaultStr) {
/**
* <p>Returns either the passed in CharSequence, or if the CharSequence is
* whitespace, empty ("") or {@code null}, the value of {@code defaultStr}.</p>
*
* @param <T> the specific kind of CharSequence
* @param str the CharSequence to check, may be null
* @param defaultStr the default CharSequence to return
* if the input is whitespace, empty ("") or {@code null}, may be null
* @return the passed in CharSequence, or the default
*/
public static <T extends CharSequence> T defaultIfBlank(final T str, final T defaultStr) {
return isBlank(str) ? defaultStr : str;
}
/**
* <p>Compares two String, returning {@code true} if they represent
* equal string, ignoring case.</p>
*
* @param str1 the first String, may be null
* @param str2 the second String, may be null
* @return {@code true} if the String are equal, case insensitive, or
* both {@code null}
*/
public static boolean equalsIgnoreCase(String str1, String str2) {
return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2);
}
@ -131,4 +174,54 @@ public class StringUtils {
}
}
}
/**
* <p>Joins the elements of the provided Collection into a single String
* containing the provided Collection of elements.</p>
*
* @param collection the collection, may be null
* @param separator the separator
* @return a single String
*/
public static String join(Collection<?> collection, String separator) {
return collection == null ? null : join(collection.iterator(), separator);
}
/**
* <p>Joins the elements of the provided Iterator into a single String
* containing the provided Iterator of elements.</p>
*
* @param iterator the iterator, may be null
* @param separator the separator
* @return a single String
*/
public static String join(Iterator<?> iterator, String separator) {
if (iterator == null) {
return null;
} else if (!iterator.hasNext()) {
return "";
} else {
Object first = iterator.next();
if (!iterator.hasNext()) {
return first == null ? "" : first.toString();
} else {
StringBuilder buf = new StringBuilder(256);
if (first != null) {
buf.append(first);
}
while (iterator.hasNext()) {
if (separator != null) {
buf.append(separator);
}
Object obj = iterator.next();
if (obj != null) {
buf.append(obj);
}
}
return buf.toString();
}
}
}
}

5
dolphinscheduler-common/src/main/resources/common.properties

@ -58,7 +58,10 @@ fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.
yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s
yarn.application.status.address=http://10.172.33.1:%s/ws/v1/cluster/apps/%s
# if custom you resourcemanager port ,you need to replace 8088 else default value.
resource.manager.httpaddress.port=8088
# job history status url when application number threshold is reached(default 10000,maybe it was set to 1000)
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s

47
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~HEAD

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
/**
* hive conf utils test
*/
public class HiveConfUtilsTest {
/**
* test is hive conf var
*/
@Test
public void testIsHiveConfVar() {
String conf = "hive.exec.script.wrapper=123";
boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
Assert.assertTrue(hiveConfVar);
conf = "hive.test.v1=v1";
hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
Assert.assertFalse(hiveConfVar);
conf = "tez.queue.name=tezQueue";
hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
Assert.assertTrue(hiveConfVar);
}
}

47
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~dev

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
/**
* hive conf utils test
*/
public class HiveConfUtilsTest {
/**
* test is hive conf var
*/
@Test
public void testIsHiveConfVar() {
String conf = "hive.exec.script.wrapper=123";
boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
Assert.assertTrue(hiveConfVar);
conf = "hive.test.v1=v1";
hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
Assert.assertFalse(hiveConfVar);
conf = "tez.queue.name=tezQueue";
hiveConfVar = HiveConfUtils.isHiveConfVar(conf);
Assert.assertTrue(hiveConfVar);
}
}

13
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java

@ -17,6 +17,9 @@
package org.apache.dolphinscheduler.common.utils;
import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
@ -80,4 +83,14 @@ public class StringUtilsTest {
defaultStr = StringUtils.defaultIfBlank("test", "defaultStr");
Assert.assertEquals("test", defaultStr);
}
@Test
public void testJoin() {
List<String> list = new ArrayList<>();
list.add("1");
list.add("3");
list.add("4");
String join = StringUtils.join(list, ",");
Assert.assertEquals("1,3,4", join);
}
}

10
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java

@ -14,17 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.common.enums.AlertType;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
/**
* alertgroup mapper interface
*/

7
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java

@ -112,4 +112,11 @@ public interface ProjectMapper extends BaseMapper<Project> {
* @return projectName and userName
*/
ProjectUser queryProjectWithUserByProcessInstanceId(@Param("processInstanceId") int processInstanceId);
/**
* query all project
* @return projectList
*/
List<Project> queryAllProject();
}

6
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml

@ -143,4 +143,8 @@
join t_ds_user u on dp.user_id = u.id
where di.id = #{processInstanceId};
</select>
</mapper>
<select id="queryAllProject" resultType="org.apache.dolphinscheduler.dao.entity.Project">
select * from t_ds_project
</select>
</mapper>

6
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java

@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -42,6 +43,7 @@ import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ -60,10 +62,10 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class);
private final ThreadPoolExecutor executor;
private final ExecutorService executor;
public LoggerRequestProcessor() {
this.executor = new ThreadPoolExecutor(4, 4, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100));
this.executor = Executors.newFixedThreadPool(Constants.CPUS * 2 + 1);
}
@Override

28
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java

@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.service.log.LogClientService;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -385,11 +386,13 @@ public class ProcessUtils {
return;
}
String cmd = String.format("kill -9 %s", getPidsStr(processId));
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", processId, cmd);
OSUtils.exeCmd(cmd);
String pidsStr = getPidsStr(processId);
if (StringUtils.isNotEmpty(pidsStr)) {
String cmd = String.format("kill -9 %s", pidsStr);
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", processId, cmd);
OSUtils.exeCmd(cmd);
}
} catch (Exception e) {
logger.error("kill task failed", e);
@ -406,7 +409,7 @@ public class ProcessUtils {
* @throws Exception exception
*/
public static String getPidsStr(int processId) throws Exception {
StringBuilder sb = new StringBuilder();
List<String> pidList = new ArrayList<>();
Matcher mat = null;
// pstree pid get sub pids
if (OSUtils.isMacOS()) {
@ -421,19 +424,22 @@ public class ProcessUtils {
if (null != mat) {
while (mat.find()) {
sb.append(mat.group(1)).append(" ");
pidList.add(mat.group(1));
}
}
return sb.toString().trim();
if (CommonUtils.isSudoEnable() && !pidList.isEmpty()) {
pidList = pidList.subList(1, pidList.size());
}
return String.join(" ", pidList).trim();
}
/**
* find logs and kill yarn tasks.
*
* @param taskExecutionContext taskExecutionContext
* @return yarn application ids
*/
public static void killYarnJob(TaskExecutionContext taskExecutionContext) {
public static List<String> killYarnJob(TaskExecutionContext taskExecutionContext) {
try {
Thread.sleep(Constants.SLEEP_TIME_MILLIS);
LogClientService logClient = null;
@ -457,11 +463,13 @@ public class ProcessUtils {
}
if (CollectionUtils.isNotEmpty(appIds)) {
cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath());
return appIds;
}
}
} catch (Exception e) {
logger.error("kill yarn job failure", e);
}
return Collections.emptyList();
}
}

11
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java

@ -124,11 +124,14 @@ public class TaskKillProcessor implements NettyRequestProcessor {
return Pair.of(true, appIds);
}
String cmd = String.format("kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()));
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd);
String pidsStr = ProcessUtils.getPidsStr(taskExecutionContext.getProcessId());
if (StringUtils.isNotEmpty(pidsStr)) {
String cmd = String.format("kill -9 %s", pidsStr);
cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd);
logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd);
OSUtils.exeCmd(cmd);
}
OSUtils.exeCmd(cmd);
} catch (Exception e) {
processFlag = false;
logger.error("kill task error", e);

7
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java

@ -408,9 +408,12 @@ public abstract class AbstractCommandExecutor {
boolean result = true;
try {
for (String appId : appIds) {
logger.info("check yarn application status, appId:{}", appId);
while (Stopper.isRunning()) {
ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId);
logger.info("appId:{}, final state:{}", appId, applicationStatus.name());
if (logger.isDebugEnabled()) {
logger.debug("check yarn application status, appId:{}, final state:{}", appId, applicationStatus.name());
}
if (applicationStatus.equals(ExecutionStatus.FAILURE)
|| applicationStatus.equals(ExecutionStatus.KILL)) {
return false;
@ -423,7 +426,7 @@ public abstract class AbstractCommandExecutor {
}
}
} catch (Exception e) {
logger.error(String.format("yarn applications: %s status failed ", appIds.toString()), e);
logger.error("yarn applications: {} , query status failed, exception:{}", StringUtils.join(appIds, ","), e);
result = false;
}
return result;

14
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java

@ -514,4 +514,18 @@ public class ProcessServiceTest {
Map<String, String> locationToMap = processService.locationToMap(locations);
Assert.assertEquals(frontTaskIdAndNameMap, locationToMap);
}
@Test
public void testCreateCommand() {
Command command = new Command();
command.setProcessDefinitionId(123);
command.setCommandParam("{\"ProcessInstanceId\":222}");
command.setCommandType(CommandType.START_PROCESS);
int mockResult = 1;
Mockito.when(commandMapper.insert(command)).thenReturn(mockResult);
int exeMethodResult = processService.createCommand(command);
Assert.assertEquals(mockResult, exeMethodResult);
Mockito.verify(commandMapper, Mockito.times(1)).insert(command);
}
}

18
dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue

@ -55,8 +55,8 @@
<el-table-column prop="modifyBy" :label="$t('Modify User')"></el-table-column>
<el-table-column :label="$t('Timing state')">
<template slot-scope="scope">
<span v-if="scope.row.scheduleReleaseState === 'OFFLINE'">{{$t('offline')}}</span>
<span v-if="scope.row.scheduleReleaseState === 'ONLINE'">{{$t('online')}}</span>
<span v-if="scope.row.scheduleReleaseState === 'OFFLINE'" class="time_offline">{{$t('offline')}}</span>
<span v-if="scope.row.scheduleReleaseState === 'ONLINE'" class="time_online">{{$t('online')}}</span>
<span v-if="!scope.row.scheduleReleaseState">-</span>
</template>
</el-table-column>
@ -522,3 +522,17 @@
components: { mVersions, mStart, mTiming, mRelatedItems }
}
</script>
<style lang="scss" rel="stylesheet/scss">
.time_online {
background-color: #5cb85c;
color: #fff;
padding: 3px;
}
.time_offline {
background-color: #ffc107;
color: #fff;
padding: 3px;
}
</style>

1
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue

@ -115,6 +115,7 @@
_go (item) {
localStore.setItem('file', `${item.alias}|${item.size}`)
if (item.directory) {
localStore.setItem('pid', `${item.id}`)
localStore.setItem('currentDir', `${item.fullName}`)
this.$router.push({ path: `/resource/file/subdirectory/${item.id}` })
} else {

1
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/_source/list.vue

@ -116,6 +116,7 @@
_go (item) {
localStore.setItem('file', `${item.alias}|${item.size}`)
if (item.directory) {
localStore.setItem('pid', `${item.id}`)
localStore.setItem('currentDir', `${item.fullName}`)
this.$router.push({ path: `/resource/udf/subUdfDirectory/${item.id}` })
}

2
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue

@ -127,11 +127,9 @@
this.deleteResource({
id: item.id
}).then(res => {
this.$refs[`poptip-${i}`][0].doClose()
this.$emit('on-update')
this.$message.success(res.msg)
}).catch(e => {
this.$refs[`poptip-${i}`][0].doClose()
this.$message.error(e.msg || '')
})
},

1
dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue

@ -121,6 +121,7 @@
this.isLeft = true
}
this.isLoading = !flag
this.searchParams.id = this.$route.params.id
this.getResourcesListP(this.searchParams).then(res => {
if (this.searchParams.pageNo > 1 && res.totalList.length === 0) {
this.searchParams.pageNo = this.searchParams.pageNo - 1

2
dolphinscheduler-ui/src/js/module/components/fileUpdate/fileChildUpdate.vue

@ -174,7 +174,7 @@
formData.append('file', this.file)
formData.append('type', this.type)
formData.append('name', this.name)
formData.append('pid', this.pid)
formData.append('pid', this.id)
formData.append('currentDir', this.currentDir)
formData.append('description', this.description)
io.post('resources/create', res => {

2
dolphinscheduler-ui/src/js/module/components/fileUpdate/resourceChildUpdate.vue

@ -174,7 +174,7 @@
formData.append('file', this.file)
formData.append('type', this.type)
formData.append('name', this.name)
formData.append('pid', this.pid)
formData.append('pid', this.id)
formData.append('currentDir', this.currentDir)
formData.append('description', this.description)
io.post('resources/create', res => {

Loading…
Cancel
Save