From e53369318bdf61f169dcbf2644caf8521b3dd536 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Tue, 30 Mar 2021 22:33:49 +0800 Subject: [PATCH 01/18] [Fix-4721][worker]The shell background starts the YARN task scenario, and the kill function is abnormal (#4722) * fix the shell starts the yarn task in the background * update StringUtils code style. * solve code smell. * add method comment in StringUtils class. * update AlertGroupMapper code style. * update AlertGroupMapperTest * update sql script test. --- .../common/utils/HadoopUtils.java | 9 +- .../common/utils/StringUtils.java | 141 ++++++++++++++++-- .../common/utils/StringUtilsTest.java | 13 ++ .../dao/mapper/AlertGroupMapper.java | 10 +- .../server/utils/ProcessUtils.java | 19 ++- .../worker/processor/TaskKillProcessor.java | 11 +- .../worker/task/AbstractCommandExecutor.java | 7 +- 7 files changed, 181 insertions(+), 29 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6a53c000fe..6dc3335f17 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -205,6 +205,7 @@ public class HadoopUtils implements Closeable { * if rmHaIds is empty, single resourcemanager enabled * if rmHaIds not empty: resourcemanager HA enabled */ + yarnEnabled = true; String appUrl = StringUtils.isEmpty(rmHaIds) ? appAddress : getAppAddress(appAddress, rmHaIds); if (StringUtils.isBlank(appUrl)) { @@ -419,7 +420,9 @@ public class HadoopUtils implements Closeable { String result = Constants.FAILED; String applicationUrl = getApplicationUrl(applicationId); - logger.info("applicationUrl={}", applicationUrl); + if (logger.isDebugEnabled()) { + logger.debug("generate yarn application url, applicationUrl={}", applicationUrl); + } String responseContent = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(applicationUrl) : HttpUtils.get(applicationUrl); if (responseContent != null) { @@ -432,7 +435,9 @@ public class HadoopUtils implements Closeable { } else { //may be in job history String jobHistoryUrl = getJobHistoryUrl(applicationId); - logger.info("jobHistoryUrl={}", jobHistoryUrl); + if (logger.isDebugEnabled()) { + logger.debug("generate yarn job history application url, jobHistoryUrl={}", jobHistoryUrl); + } responseContent = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false) ? KerberosHttpClient.get(jobHistoryUrl) : HttpUtils.get(jobHistoryUrl); if (null != responseContent) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index f506f76627..ffa783316b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -17,48 +17,169 @@ package org.apache.dolphinscheduler.common.utils; +import java.util.Collection; +import java.util.Iterator; + +/** + * java.lang.String utils class + */ public class StringUtils { + /** + * The empty String {@code ""}. + */ public static final String EMPTY = ""; private StringUtils() { throw new UnsupportedOperationException("Construct StringUtils"); } + /** + *

Checks if a CharSequence is empty ("") or null.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is empty or null + */ public static boolean isEmpty(final CharSequence cs) { return cs == null || cs.length() == 0; } + /** + *

Checks if a CharSequence is not empty ("") and not null.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is not empty and not null + */ public static boolean isNotEmpty(final CharSequence cs) { return !isEmpty(cs); } - public static boolean isBlank(String str) { + /** + *

Checks if a CharSequence is empty (""), null or whitespace only.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is null, empty or whitespace only + */ + public static boolean isBlank(final CharSequence cs) { int strLen; - if (str != null && (strLen = str.length()) != 0) { - for (int i = 0; i < strLen; ++i) { - if (!Character.isWhitespace(str.charAt(i))) { - return false; - } + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(cs.charAt(i))) { + return false; } } return true; + } + /** + *

Checks if a CharSequence is not empty (""), not null and not whitespace only.

+ * + * @param cs the CharSequence to check, may be null + * @return {@code true} if the CharSequence is not empty and not null and not whitespace only + */ + public static boolean isNotBlank(final CharSequence cs) { + return !isBlank(cs); } - public static boolean isNotBlank(String s) { - return !isBlank(s); + /** + *

Replace all strings matching the regular expression \t \n \r with _

+ * + * @param src the String , may be null + * @return the string that has been replaced + */ + public static String replaceNRTtoUnderline(String src) { + return isBlank(src) ? src : src.replaceAll("[\n|\r|\t]", "_"); } - public static String trim(String str) { + /** + *

Removes control characters (char <= 32) from both + * ends of this String, handling {@code null} by returning + * {@code null}.

+ * + * @param str the String to be trimmed, may be null + * @return the trimmed string, {@code null} if null String input + */ + public static String trim(final String str) { return str == null ? null : str.trim(); } - public static String defaultIfBlank(String str, String defaultStr) { + /** + *

Returns either the passed in CharSequence, or if the CharSequence is + * whitespace, empty ("") or {@code null}, the value of {@code defaultStr}.

+ * + * @param the specific kind of CharSequence + * @param str the CharSequence to check, may be null + * @param defaultStr the default CharSequence to return + * if the input is whitespace, empty ("") or {@code null}, may be null + * @return the passed in CharSequence, or the default + */ + public static T defaultIfBlank(final T str, final T defaultStr) { return isBlank(str) ? defaultStr : str; } + /** + *

Compares two String, returning {@code true} if they represent + * equal string, ignoring case.

+ * + * @param str1 the first String, may be null + * @param str2 the second String, may be null + * @return {@code true} if the String are equal, case insensitive, or + * both {@code null} + */ public static boolean equalsIgnoreCase(String str1, String str2) { return str1 == null ? str2 == null : str1.equalsIgnoreCase(str2); } + + /** + *

Joins the elements of the provided Collection into a single String + * containing the provided Collection of elements.

+ * + * @param collection the collection, may be null + * @param separator the separator + * @return a single String + */ + public static String join(Collection collection, String separator) { + return collection == null ? null : join(collection.iterator(), separator); + } + + /** + *

Joins the elements of the provided Iterator into a single String + * containing the provided Iterator of elements.

+ * + * @param iterator the iterator, may be null + * @param separator the separator + * @return a single String + */ + public static String join(Iterator iterator, String separator) { + if (iterator == null) { + return null; + } else if (!iterator.hasNext()) { + return ""; + } else { + Object first = iterator.next(); + if (!iterator.hasNext()) { + return first == null ? "" : first.toString(); + } else { + StringBuilder buf = new StringBuilder(256); + if (first != null) { + buf.append(first); + } + + while (iterator.hasNext()) { + if (separator != null) { + buf.append(separator); + } + + Object obj = iterator.next(); + if (obj != null) { + buf.append(obj); + } + } + return buf.toString(); + } + } + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java index 3f5aeda3f9..2e59750fe2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringUtilsTest.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.common.utils; +import java.util.ArrayList; +import java.util.List; + import org.junit.Assert; import org.junit.Test; @@ -80,4 +83,14 @@ public class StringUtilsTest { defaultStr = StringUtils.defaultIfBlank("test", "defaultStr"); Assert.assertEquals("test", defaultStr); } + + @Test + public void testJoin() { + List list = new ArrayList<>(); + list.add("1"); + list.add("3"); + list.add("4"); + String join = StringUtils.join(list, ","); + Assert.assertEquals("1,3,4", join); + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java index 8026b9a89f..b8f4188fc7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java @@ -14,17 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.dao.entity.AlertGroup; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + /** * alertgroup mapper interface */ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index aca2afedab..9e5743eaf0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.service.log.LogClientService; import java.io.File; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -385,11 +386,13 @@ public class ProcessUtils { return; } - String cmd = String.format("kill -9 %s", getPidsStr(processId)); - cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); - logger.info("process id:{}, cmd:{}", processId, cmd); - - OSUtils.exeCmd(cmd); + String pidsStr = getPidsStr(processId); + if (StringUtils.isNotEmpty(pidsStr)) { + String cmd = String.format("kill -9 %s", pidsStr); + cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); + logger.info("process id:{}, cmd:{}", processId, cmd); + OSUtils.exeCmd(cmd); + } } catch (Exception e) { logger.error("kill task failed", e); @@ -430,10 +433,10 @@ public class ProcessUtils { /** * find logs and kill yarn tasks. - * * @param taskExecutionContext taskExecutionContext + * @return yarn application ids */ - public static void killYarnJob(TaskExecutionContext taskExecutionContext) { + public static List killYarnJob(TaskExecutionContext taskExecutionContext) { try { Thread.sleep(Constants.SLEEP_TIME_MILLIS); LogClientService logClient = null; @@ -457,11 +460,13 @@ public class ProcessUtils { } if (CollectionUtils.isNotEmpty(appIds)) { cancelApplication(appIds, logger, taskExecutionContext.getTenantCode(), taskExecutionContext.getExecutePath()); + return appIds; } } } catch (Exception e) { logger.error("kill yarn job failure", e); } + return Collections.emptyList(); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java index 84109ccbc8..8cbf0471b2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java @@ -124,11 +124,14 @@ public class TaskKillProcessor implements NettyRequestProcessor { return Pair.of(true, appIds); } - String cmd = String.format("kill -9 %s", ProcessUtils.getPidsStr(taskExecutionContext.getProcessId())); - cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); - logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd); + String pidsStr = ProcessUtils.getPidsStr(taskExecutionContext.getProcessId()); + if (StringUtils.isNotEmpty(pidsStr)) { + String cmd = String.format("kill -9 %s", pidsStr); + cmd = OSUtils.getSudoCmd(taskExecutionContext.getTenantCode(), cmd); + logger.info("process id:{}, cmd:{}", taskExecutionContext.getProcessId(), cmd); + OSUtils.exeCmd(cmd); + } - OSUtils.exeCmd(cmd); } catch (Exception e) { processFlag = false; logger.error("kill task error", e); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 88af2d7f2c..788d5441df 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -408,9 +408,12 @@ public abstract class AbstractCommandExecutor { boolean result = true; try { for (String appId : appIds) { + logger.info("check yarn application status, appId:{}", appId); while (Stopper.isRunning()) { ExecutionStatus applicationStatus = HadoopUtils.getInstance().getApplicationStatus(appId); - logger.info("appId:{}, final state:{}", appId, applicationStatus.name()); + if (logger.isDebugEnabled()) { + logger.debug("check yarn application status, appId:{}, final state:{}", appId, applicationStatus.name()); + } if (applicationStatus.equals(ExecutionStatus.FAILURE) || applicationStatus.equals(ExecutionStatus.KILL)) { return false; @@ -423,7 +426,7 @@ public abstract class AbstractCommandExecutor { } } } catch (Exception e) { - logger.error(String.format("yarn applications: %s status failed ", appIds.toString()), e); + logger.error("yarn applications: {} , query status failed, exception:{}", StringUtils.join(appIds, ","), e); result = false; } return result; From 817328ee8f401e22508ac4404a20270abc17b192 Mon Sep 17 00:00:00 2001 From: wanghong1314 <498213175@qq.com> Date: Wed, 31 Mar 2021 20:50:12 +0800 Subject: [PATCH 02/18] [Improvement-5172][UI]process list add Timing state add online/offline marking color (#5190) Co-authored-by: wanghong --- .../definition/pages/list/_source/list.vue | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 882bd67ddd..435942da5c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -55,8 +55,8 @@ @@ -522,3 +522,17 @@ components: { mVersions, mStart, mTiming, mRelatedItems } } + + From f20bb54896a9ac7b65c20b0cc7b0a744d1741155 Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Thu, 1 Apr 2021 20:34:49 +0800 Subject: [PATCH 03/18] [Improvement][API] Improvement query all project list (#5169) (#5192) --- .../api/service/impl/ProjectServiceImpl.java | 9 ++------- .../dolphinscheduler/api/service/ProjectServiceTest.java | 3 +-- .../dolphinscheduler/dao/mapper/ProjectMapper.java | 6 ++++++ .../apache/dolphinscheduler/dao/mapper/ProjectMapper.xml | 4 ++++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index c2e85cf612..6b25993098 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -447,19 +447,14 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic } /** - * query all project list that have one or more process definitions. + * query all project list * * @return project list */ @Override public Map queryAllProjectList() { Map result = new HashMap<>(); - List projects = new ArrayList<>(); - - List projectIds = processDefinitionMapper.listProjectIds(); - if (CollectionUtils.isNotEmpty(projectIds)) { - projects = projectMapper.selectBatchIds(projectIds); - } + List projects = projectMapper.queryAllProject(); result.put(Constants.DATA_LIST, projects); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java index 3cdf7142e6..7b70206df8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java @@ -322,8 +322,7 @@ public class ProjectServiceTest { @Test public void testQueryAllProjectList() { - Mockito.when(processDefinitionMapper.listProjectIds()).thenReturn(getProjectIds()); - Mockito.when(projectMapper.selectBatchIds(getProjectIds())).thenReturn(getList()); + Mockito.when(projectMapper.queryAllProject()).thenReturn(getList()); Map result = projectService.queryAllProjectList(); logger.info(result.toString()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java index 872d53fd92..b84272cc89 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java @@ -92,4 +92,10 @@ public interface ProjectMapper extends BaseMapper { */ ProjectUser queryProjectWithUserByProcessInstanceId(@Param("processInstanceId") int processInstanceId); + /** + * query all project + * @return projectList + */ + List queryAllProject(); + } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml index a6d72d0d2d..74844d41a8 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.xml @@ -109,4 +109,8 @@ join t_ds_user u on dp.user_id = u.id where di.id = #{processInstanceId}; + + From 6b565f0aed07cd328702d9fc1bec99e682c7525f Mon Sep 17 00:00:00 2001 From: guohaozhang Date: Thu, 1 Apr 2021 20:44:03 +0800 Subject: [PATCH 04/18] [Fix-#5124] fix change fileName when reupload file (#5125) --- .../dolphinscheduler/api/service/impl/ResourcesServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 4b7cc592ac..80923114b6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -367,7 +367,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe resource.setDescription(desc); resource.setUpdateTime(now); if (file != null) { - resource.setFileName(file.getOriginalFilename()); resource.setSize(file.getSize()); } From 5d7dc1cb2aff2147b09b13ed002cbe54f3478a2c Mon Sep 17 00:00:00 2001 From: Hayden Zhou Date: Thu, 1 Apr 2021 20:50:14 +0800 Subject: [PATCH 05/18] [Improvement][API]optimize tenant code validation (#5093) * optimize tenant code validation * fix code style and test err * fix test error, and remove duplicate tenant code validate --- .../api/service/impl/TenantServiceImpl.java | 7 +------ .../api/utils/RegexUtils.java | 12 +++++++++++ .../api/controller/TenantControllerTest.java | 4 ++-- .../api/service/TenantServiceTest.java | 4 ++-- .../api/utils/RegexUtilsTest.java | 21 +++++++++++++++++++ 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 8a3a945d10..dcdc833dcc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -90,7 +90,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService return result; } - if (RegexUtils.isNumeric(tenantCode)) { + if (!RegexUtils.isValidLinuxUserName(tenantCode)) { putMsg(result, Status.CHECK_OS_TENANT_CODE_ERROR); return result; } @@ -102,11 +102,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService Tenant tenant = new Tenant(); Date now = new Date(); - - if (!tenantCode.matches("^[0-9a-zA-Z_.-]{1,}$") || tenantCode.startsWith("-") || tenantCode.startsWith(".")) { - putMsg(result, Status.VERIFY_OS_TENANT_CODE_ERROR); - return result; - } tenant.setTenantCode(tenantCode); tenant.setQueueId(queueId); tenant.setDescription(desc); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java index 482cb55306..2e43d2bdaf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java @@ -30,6 +30,8 @@ public class RegexUtils { */ private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$"; + private static final String LINUX_USERNAME_PATTERN = "[a-z_][a-z\\d_]{0,30}"; + private RegexUtils() { } @@ -45,6 +47,16 @@ public class RegexUtils { return isNum.matches(); } + /** + * check if the input is a valid linux username + * @param str input + * @return boolean + */ + public static boolean isValidLinuxUserName(String str) { + Pattern pattern = Pattern.compile(LINUX_USERNAME_PATTERN); + return pattern.matcher(str).matches(); + } + public static String escapeNRT(String str) { // Logging should not be vulnerable to injection attacks: Replace pattern-breaking characters if (str != null && !str.isEmpty()) { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java index c9463a82d9..c9524e8158 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java @@ -45,7 +45,7 @@ public class TenantControllerTest extends AbstractControllerTest{ @Test public void testCreateTenant() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("tenantCode","tenantCode"); + paramsMap.add("tenantCode","hayden"); paramsMap.add("queueId","1"); paramsMap.add("description","tenant description"); @@ -124,7 +124,7 @@ public class TenantControllerTest extends AbstractControllerTest{ @Test public void testVerifyTenantCodeExists() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("tenantCode", "tenantCode"); + paramsMap.add("tenantCode", "hayden"); MvcResult mvcResult = mockMvc.perform(get("/tenant/verify-tenant-code") .header(SESSION_ID, sessionId) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index ee85d260cc..8889fa6b07 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -73,7 +73,7 @@ public class TenantServiceTest { @Mock private UserMapper userMapper; - private static final String tenantCode = "TenantServiceTest"; + private static final String tenantCode = "hayden"; @Test public void testCreateTenant() { @@ -85,7 +85,7 @@ public class TenantServiceTest { Map result = tenantService.createTenant(getLoginUser(), "%!1111", 1, "TenantServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.VERIFY_OS_TENANT_CODE_ERROR, result.get(Constants.STATUS)); + Assert.assertEquals(Status.CHECK_OS_TENANT_CODE_ERROR, result.get(Constants.STATUS)); //check exist result = tenantService.createTenant(loginUser, tenantCode, 1, "TenantServiceTest"); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java index 7ccfdfdb38..067dfeb17d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java @@ -36,6 +36,27 @@ public class RegexUtilsTest { Assert.assertFalse(numeric2); } + @Test + public void testIsValidLinuxUserName() { + String name1 = "10000"; + Assert.assertFalse(RegexUtils.isValidLinuxUserName(name1)); + + String name2 = "00hayden"; + Assert.assertFalse(RegexUtils.isValidLinuxUserName(name2)); + + String name3 = "hayde123456789123456789123456789"; + Assert.assertFalse(RegexUtils.isValidLinuxUserName(name3)); + + String name4 = "hayd123456789123456789123456789"; + Assert.assertTrue(RegexUtils.isValidLinuxUserName(name4)); + + String name5 = "h"; + Assert.assertTrue(RegexUtils.isValidLinuxUserName(name5)); + + String name6 = "hayden"; + Assert.assertTrue(RegexUtils.isValidLinuxUserName(name6)); + } + @Test public void testEscapeNRT() { String result1 = RegexUtils.escapeNRT("abc\n"); From e8c9c33d7e45b361c3c25085b01d3d78b6b11d90 Mon Sep 17 00:00:00 2001 From: JuFeng Li <920347627@qq.com> Date: Thu, 1 Apr 2021 20:57:46 +0800 Subject: [PATCH 06/18] [Fix-5055][UI] Fix can not upload file to correct dir (#5056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix udf resource can not upload to the correct dir,and when delete udf resource can not refresh page Co-authored-by: 李巨丰 --- .../resource/pages/file/pages/subdirectory/_source/list.vue | 1 + .../pages/resource/pages/udf/pages/resource/_source/list.vue | 1 + .../resource/pages/udf/pages/subUdfDirectory/_source/list.vue | 2 -- .../pages/resource/pages/udf/pages/subUdfDirectory/index.vue | 1 + .../src/js/module/components/fileUpdate/fileChildUpdate.vue | 2 +- .../src/js/module/components/fileUpdate/resourceChildUpdate.vue | 2 +- 6 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue index f0098578a7..c51aded8b5 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/file/pages/subdirectory/_source/list.vue @@ -115,6 +115,7 @@ _go (item) { localStore.setItem('file', `${item.alias}|${item.size}`) if (item.directory) { + localStore.setItem('pid', `${item.id}`) localStore.setItem('currentDir', `${item.fullName}`) this.$router.push({ path: `/resource/file/subdirectory/${item.id}` }) } else { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/_source/list.vue index f25178c09e..24bf71228d 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/resource/_source/list.vue @@ -116,6 +116,7 @@ _go (item) { localStore.setItem('file', `${item.alias}|${item.size}`) if (item.directory) { + localStore.setItem('pid', `${item.id}`) localStore.setItem('currentDir', `${item.fullName}`) this.$router.push({ path: `/resource/udf/subUdfDirectory/${item.id}` }) } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue index 8522190c1b..17d0d2ad05 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue @@ -127,11 +127,9 @@ this.deleteResource({ id: item.id }).then(res => { - this.$refs[`poptip-${i}`][0].doClose() this.$emit('on-update') this.$message.success(res.msg) }).catch(e => { - this.$refs[`poptip-${i}`][0].doClose() this.$message.error(e.msg || '') }) }, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue index 5de6fdb1c0..654d5fbee9 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/index.vue @@ -121,6 +121,7 @@ this.isLeft = true } this.isLoading = !flag + this.searchParams.id = this.$route.params.id this.getResourcesListP(this.searchParams).then(res => { if (this.searchParams.pageNo > 1 && res.totalList.length === 0) { this.searchParams.pageNo = this.searchParams.pageNo - 1 diff --git a/dolphinscheduler-ui/src/js/module/components/fileUpdate/fileChildUpdate.vue b/dolphinscheduler-ui/src/js/module/components/fileUpdate/fileChildUpdate.vue index a984af6854..369a1613a6 100644 --- a/dolphinscheduler-ui/src/js/module/components/fileUpdate/fileChildUpdate.vue +++ b/dolphinscheduler-ui/src/js/module/components/fileUpdate/fileChildUpdate.vue @@ -174,7 +174,7 @@ formData.append('file', this.file) formData.append('type', this.type) formData.append('name', this.name) - formData.append('pid', this.pid) + formData.append('pid', this.id) formData.append('currentDir', this.currentDir) formData.append('description', this.description) io.post('resources/create', res => { diff --git a/dolphinscheduler-ui/src/js/module/components/fileUpdate/resourceChildUpdate.vue b/dolphinscheduler-ui/src/js/module/components/fileUpdate/resourceChildUpdate.vue index 4bc438f31e..fedb2b2392 100755 --- a/dolphinscheduler-ui/src/js/module/components/fileUpdate/resourceChildUpdate.vue +++ b/dolphinscheduler-ui/src/js/module/components/fileUpdate/resourceChildUpdate.vue @@ -174,7 +174,7 @@ formData.append('file', this.file) formData.append('type', this.type) formData.append('name', this.name) - formData.append('pid', this.pid) + formData.append('pid', this.id) formData.append('currentDir', this.currentDir) formData.append('description', this.description) io.post('resources/create', res => { From 97fb08f0ced20a4b9563725ed9e9600e5ea5722b Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Fri, 2 Apr 2021 14:24:37 +0800 Subject: [PATCH 07/18] [Improvement][LogServer] Thread pool parameter adjustment #5161 (#5163) * [Improvement][LogServer] Thread pool parameter adjustment #5161 * change log request processor thread num 2 * cpuCores + 1 --- .../dolphinscheduler/server/log/LoggerRequestProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java index c60a4479c1..c9350bf8fb 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; +import org.apache.dolphinscheduler.remote.utils.Constants; import java.io.ByteArrayOutputStream; import java.io.File; @@ -42,6 +43,7 @@ import java.nio.file.Paths; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -60,10 +62,10 @@ public class LoggerRequestProcessor implements NettyRequestProcessor { private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class); - private final ThreadPoolExecutor executor; + private final ExecutorService executor; public LoggerRequestProcessor() { - this.executor = new ThreadPoolExecutor(4, 4, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100)); + this.executor = Executors.newFixedThreadPool(Constants.CPUS * 2 + 1); } @Override From 4a1303b625f3425550636a522ad2e38bd8cae1b7 Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Tue, 6 Apr 2021 10:47:53 +0800 Subject: [PATCH 08/18] [Improvement][Worker] Kill task error(#5199) (#5212) --- .../dolphinscheduler/server/utils/ProcessUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 9e5743eaf0..f7a9991a08 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -409,7 +409,7 @@ public class ProcessUtils { * @throws Exception exception */ public static String getPidsStr(int processId) throws Exception { - StringBuilder sb = new StringBuilder(); + List pidList = new ArrayList<>(); Matcher mat = null; // pstree pid get sub pids if (OSUtils.isMacOS()) { @@ -424,11 +424,14 @@ public class ProcessUtils { if (null != mat) { while (mat.find()) { - sb.append(mat.group(1)).append(" "); + pidList.add(mat.group(1)); } } - return sb.toString().trim(); + if (CommonUtils.isSudoEnable() && !pidList.isEmpty()) { + pidList = pidList.subList(1, pidList.size()); + } + return String.join(" ", pidList).trim(); } /** From 2f1f193ba85abb7d4738f4f10ad05e6d510132c5 Mon Sep 17 00:00:00 2001 From: xiaoyu Date: Wed, 7 Apr 2021 16:17:21 +0800 Subject: [PATCH 09/18] Update common.properties (#5223) --- dolphinscheduler-common/src/main/resources/common.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index e46dd15756..0fbf5f4091 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -60,6 +60,9 @@ yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s +# if custom you resourcemanager port ,you need to replace 8088 else default value. +resource.manager.httpaddress.port=8088 + # job history status url when application number threshold is reached(default 10000,maybe it was set to 1000) yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s From 5c898e38ed62b0f8d034d1e910be4b45692dae8e Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Thu, 8 Apr 2021 18:58:43 +0800 Subject: [PATCH 10/18] [FIX-#5233][API] Fix bug ofREPEAT RUNNING processInstance throw exceprion (#5234) * fix bug of REPEAT RUNNING throw exception Co-authored-by: wangxj --- .../api/service/impl/ExecutorServiceImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java index 8e75eb9c5c..9747127ef7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java @@ -24,7 +24,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODE_ import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS; import static org.apache.dolphinscheduler.common.Constants.MAX_TASK_TIMEOUT; -import org.apache.commons.collections.MapUtils; import org.apache.dolphinscheduler.api.enums.ExecuteType; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.ExecutorService; @@ -57,6 +56,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; +import org.apache.commons.collections.MapUtils; + import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -266,7 +267,10 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ Map commandMap = JSONUtils.toMap(processInstance.getCommandParam(), String.class, Object.class); String startParams = null; if (MapUtils.isNotEmpty(commandMap) && executeType == ExecuteType.REPEAT_RUNNING) { - startParams = (commandMap.get(Constants.CMD_PARAM_START_PARAMS)).toString(); + Object startParamsJson = commandMap.get(Constants.CMD_PARAM_START_PARAMS); + if (startParamsJson != null) { + startParams = startParamsJson.toString(); + } } switch (executeType) { From f3195fce358ba2c46319048f16f84b63d7885e06 Mon Sep 17 00:00:00 2001 From: DS Date: Fri, 9 Apr 2021 01:03:04 +0800 Subject: [PATCH 11/18] Update README.md (#5238) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index faa7eefb4d..e4be878d3c 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ Dolphin Scheduler Official Website [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) +[![Twitter Follow](https://img.shields.io/twitter/follow/dolphinschedule.svg?style=social&label=Follow)](https://twitter.com/dolphinschedule) +[![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) + [![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler) From 641dedd21e2b5d59bb13ba283ea4c06e5223b4ae Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Sat, 10 Apr 2021 13:44:36 +0800 Subject: [PATCH 12/18] fix dag empty msg (#5245) --- .../java/org/apache/dolphinscheduler/api/enums/Status.java | 1 + .../api/service/impl/ProcessDefinitionServiceImpl.java | 4 ++-- .../api/service/ProcessDefinitionServiceTest.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 062b61e2b9..298dbe6770 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -262,6 +262,7 @@ public enum Status { EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028, "export process definition by id error", "导出工作流定义错误"), BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR(50028, "batch export process definition by ids error", "批量导出工作流定义错误"), IMPORT_PROCESS_DEFINE_ERROR(50029, "import process definition error", "导入工作流定义错误"), + PROCESS_DAG_IS_EMPTY(50030, "process dag can not be empty", "工作流dag不能为空"), HDFS_NOT_STARTUP(60001, "hdfs not startup", "hdfs未启用"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 4ab43fb37b..8358bd5628 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -1168,9 +1168,9 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro // Check whether the task node is normal List taskNodes = processData.getTasks(); - if (taskNodes == null) { + if (CollectionUtils.isEmpty(taskNodes)) { logger.error("process node info is empty"); - putMsg(result, Status.DATA_IS_NULL, processDefinitionJson); + putMsg(result, Status.PROCESS_DAG_IS_EMPTY); return result; } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index a4b3527e40..26d0b8839d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -679,7 +679,7 @@ public class ProcessDefinitionServiceTest { // task empty processData.setTasks(null); Map taskNotEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); - Assert.assertEquals(Status.DATA_IS_NULL, taskNotEmptyRes.get(Constants.STATUS)); + Assert.assertEquals(Status.PROCESS_DAG_IS_EMPTY, taskNotEmptyRes.get(Constants.STATUS)); // task cycle String processDefinitionJsonCycle = CYCLE_SHELL_JSON; From d4812f0a348733c59fc23d6e55e53e180a81461e Mon Sep 17 00:00:00 2001 From: "Jonathan.Wei" Date: Sun, 11 Apr 2021 13:26:27 +0800 Subject: [PATCH 13/18] [Fix-5250][API] Fixed data source parameter type attribute being null (#5251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update DataSourceServiceImpl.java Fixed data source parameter type attribute being null * Fixed test case issues related to deleting the type parameter Co-authored-by: 魏基勇 --- .../api/service/impl/DataSourceServiceImpl.java | 1 - .../api/service/DataSourceServiceTest.java | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 474221562d..1a2ea7b57d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -488,7 +488,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource separator = ";"; } - parameterMap.put(TYPE, connectType); parameterMap.put(Constants.ADDRESS, address); parameterMap.put(Constants.DATABASE, database); parameterMap.put(Constants.JDBC_URL, jdbcUrl); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index 5e50ff77b0..f127b9d98b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -281,7 +281,7 @@ public class DataSourceServiceTest { public void buildParameter() { String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im" , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "", "", "", ""); - String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\"," + String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\"," + "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; Assert.assertEquals(expected, param); @@ -290,7 +290,7 @@ public class DataSourceServiceTest { PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test"); param = dataSourceService.buildParameter(DbType.HIVE, "192.168.9.1", "10000", "im" , "hive/hdfs-mycluster@ESZ.COM", "test", "test", null, "", "/opt/krb5.conf", "test2/hdfs-mycluster@ESZ.COM", "/opt/hdfs.headless.keytab"); - expected = "{\"type\":null,\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\"," + expected = "{\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\"," + "\"user\":\"test\",\"password\":\"test\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\"," + "\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}"; Assert.assertEquals(expected, param); @@ -303,14 +303,14 @@ public class DataSourceServiceTest { String other = "{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\"}"; String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" , "", "test", "123456", null, other, "", "", ""); - String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," + String expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," + "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; Assert.assertEquals(expected, param); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" , "", "test", "123456", null, "", "", "", ""); - expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," + expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," + "\"user\":\"test\",\"password\":\"123456\"}"; Assert.assertEquals(expected, param); } From 73e917d3c179b67afd0f3fbd2902d683dbed172d Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Sun, 11 Apr 2021 14:07:58 +0800 Subject: [PATCH 14/18] Clean up useless files. (#5252) --- .../common/utils/HiveConfUtilsTest.java~HEAD | 47 ------------------- .../common/utils/HiveConfUtilsTest.java~dev | 47 ------------------- 2 files changed, 94 deletions(-) delete mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~HEAD delete mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~dev diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~HEAD b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~HEAD deleted file mode 100644 index 85912d5ff6..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~HEAD +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -import org.junit.Assert; -import org.junit.Test; - -/** - * hive conf utils test - */ -public class HiveConfUtilsTest { - - /** - * test is hive conf var - */ - @Test - public void testIsHiveConfVar() { - - String conf = "hive.exec.script.wrapper=123"; - boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertTrue(hiveConfVar); - - conf = "hive.test.v1=v1"; - hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertFalse(hiveConfVar); - - conf = "tez.queue.name=tezQueue"; - hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertTrue(hiveConfVar); - - } -} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~dev b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~dev deleted file mode 100644 index 85912d5ff6..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HiveConfUtilsTest.java~dev +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -import org.junit.Assert; -import org.junit.Test; - -/** - * hive conf utils test - */ -public class HiveConfUtilsTest { - - /** - * test is hive conf var - */ - @Test - public void testIsHiveConfVar() { - - String conf = "hive.exec.script.wrapper=123"; - boolean hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertTrue(hiveConfVar); - - conf = "hive.test.v1=v1"; - hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertFalse(hiveConfVar); - - conf = "tez.queue.name=tezQueue"; - hiveConfVar = HiveConfUtils.isHiveConfVar(conf); - Assert.assertTrue(hiveConfVar); - - } -} From 039d517c1358c197629d54c580ba6e1b356d551d Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Sun, 11 Apr 2021 14:44:20 +0800 Subject: [PATCH 15/18] [Improvement][API] the QueryAuthorizedProject interface adds permission judgment (#5170) (#5184) --- .../api/service/impl/ProjectServiceImpl.java | 8 ++------ .../api/service/ProjectServiceTest.java | 10 +++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 6b25993098..3365b940f5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -301,7 +301,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic @Override public Map queryUnauthorizedProject(User loginUser, Integer userId) { Map result = new HashMap<>(); - if (isNotAdmin(loginUser, result)) { + if (loginUser.getId() != userId && isNotAdmin(loginUser, result)) { return result; } /** @@ -353,7 +353,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic public Map queryAuthorizedProject(User loginUser, Integer userId) { Map result = new HashMap<>(); - if (isNotAdmin(loginUser, result)) { + if (loginUser.getId() != userId && isNotAdmin(loginUser, result)) { return result; } @@ -374,10 +374,6 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic public Map queryProjectCreatedByUser(User loginUser) { Map result = new HashMap<>(); - if (isNotAdmin(loginUser, result)) { - return result; - } - List projects = projectMapper.queryProjectCreatedByUser(loginUser.getId()); result.put(Constants.DATA_LIST, projects); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java index 7b70206df8..ebb6026f63 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java @@ -276,6 +276,10 @@ public class ProjectServiceTest { List projects = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + loginUser.setUserType(UserType.GENERAL_USER); + result = projectService.queryAuthorizedProject(loginUser, loginUser.getId()); + projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); } @Test @@ -284,14 +288,10 @@ public class ProjectServiceTest { User loginUser = getLoginUser(); Mockito.when(projectMapper.queryProjectCreatedByUser(1)).thenReturn(getList()); - //USER_NO_OPERATION_PERM - Map result = projectService.queryProjectCreatedByUser(loginUser); - logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //success loginUser.setUserType(UserType.ADMIN_USER); - result = projectService.queryProjectCreatedByUser(loginUser); + Map result = projectService.queryProjectCreatedByUser(loginUser); logger.info(result.toString()); List projects = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); From 3e64f43138d757720be9e167075c6fab4c0d4f70 Mon Sep 17 00:00:00 2001 From: Bruce Lau Date: Mon, 12 Apr 2021 18:55:00 +0800 Subject: [PATCH 16/18] [Test-4124][service] Add the testCreateCommand test of ProcessService Class (#5257) * [uint test] Improve code test coverage of ProcessService class * [Test-4124][service] Add the testCreateCommand test of ProcessService Class Co-authored-by: liuziying --- .../service/process/ProcessServiceTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index 999ca46a4e..a693c2e6d8 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -464,4 +464,17 @@ public class ProcessServiceTest { processService.changeOutParam(result,taskInstance); } + @Test + public void testCreateCommand() { + Command command = new Command(); + command.setProcessDefinitionId(123); + command.setCommandParam("{\"ProcessInstanceId\":222}"); + command.setCommandType(CommandType.START_PROCESS); + int mockResult = 1; + Mockito.when(commandMapper.insert(command)).thenReturn(mockResult); + int exeMethodResult = processService.createCommand(command); + Assert.assertEquals(mockResult, exeMethodResult); + Mockito.verify(commandMapper, Mockito.times(1)).insert(command); + } + } From 0eb8c0c234f92c075fa0b4df01feb85ae088e153 Mon Sep 17 00:00:00 2001 From: didiaode18 <563646039@qq.com> Date: Mon, 12 Apr 2021 19:32:57 +0800 Subject: [PATCH 17/18] [Improvement][common]Task status error --- .../common/utils/HadoopUtils.java | 49 +++++++++---------- .../src/main/resources/common.properties | 2 +- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6dc3335f17..b76f2b1977 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -214,7 +214,8 @@ public class HadoopUtils implements Closeable { if (logger.isDebugEnabled()) { logger.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId); } - return String.format(appUrl, applicationId); + String activeResourceManagerPort = String.valueOf(PropertyUtils.getInt(Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT, 8088)); + return String.format(appUrl, activeResourceManagerPort, applicationId); } public String getJobHistoryUrl(String applicationId) { @@ -247,7 +248,7 @@ public class HadoopUtils implements Closeable { * * @param hdfsFilePath hdfs file path * @param skipLineNums skip line numbers - * @param limit read how many lines + * @param limit read how many lines * @return content of file * @throws IOException errors */ @@ -282,10 +283,10 @@ public class HadoopUtils implements Closeable { /** * copy files between FileSystems * - * @param srcPath source hdfs path - * @param dstPath destination hdfs path + * @param srcPath source hdfs path + * @param dstPath destination hdfs path * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file + * @param overwrite whether to overwrite an existing file * @return if success or not * @throws IOException errors */ @@ -297,10 +298,10 @@ public class HadoopUtils implements Closeable { * the src file is on the local disk. Add it to FS at * the given dst name. * - * @param srcFile local file - * @param dstHdfsPath destination hdfs path + * @param srcFile local file + * @param dstHdfsPath destination hdfs path * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file + * @param overwrite whether to overwrite an existing file * @return if success or not * @throws IOException errors */ @@ -317,9 +318,9 @@ public class HadoopUtils implements Closeable { * copy hdfs file to local * * @param srcHdfsFilePath source hdfs file path - * @param dstFile destination file - * @param deleteSource delete source - * @param overwrite overwrite + * @param dstFile destination file + * @param deleteSource delete source + * @param overwrite overwrite * @return result of copy hdfs file to local * @throws IOException errors */ @@ -348,9 +349,9 @@ public class HadoopUtils implements Closeable { * delete a file * * @param hdfsFilePath the path to delete. - * @param recursive if path is a directory and set to - * true, the directory is deleted else throws an exception. In - * case of a file the recursive can be set to either true or false. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. * @return true if delete is successful else false. * @throws IOException errors */ @@ -487,7 +488,7 @@ public class HadoopUtils implements Closeable { /** * hdfs resource dir * - * @param tenantCode tenant code + * @param tenantCode tenant code * @param resourceType resource type * @return hdfs resource dir */ @@ -515,7 +516,7 @@ public class HadoopUtils implements Closeable { * hdfs user dir * * @param tenantCode tenant code - * @param userId user id + * @param userId user id * @return hdfs resource dir */ public static String getHdfsUserDir(String tenantCode, int userId) { @@ -536,8 +537,8 @@ public class HadoopUtils implements Closeable { * get hdfs file name * * @param resourceType resource type - * @param tenantCode tenant code - * @param fileName file name + * @param tenantCode tenant code + * @param fileName file name * @return hdfs file name */ public static String getHdfsFileName(ResourceType resourceType, String tenantCode, String fileName) { @@ -551,7 +552,7 @@ public class HadoopUtils implements Closeable { * get absolute path and name for resource file on hdfs * * @param tenantCode tenant code - * @param fileName file name + * @param fileName file name * @return get absolute path and name for file on hdfs */ public static String getHdfsResourceFileName(String tenantCode, String fileName) { @@ -565,7 +566,7 @@ public class HadoopUtils implements Closeable { * get absolute path and name for udf file on hdfs * * @param tenantCode tenant code - * @param fileName file name + * @param fileName file name * @return get absolute path and name for udf file on hdfs */ public static String getHdfsUdfFileName(String tenantCode, String fileName) { @@ -587,7 +588,7 @@ public class HadoopUtils implements Closeable { * getAppAddress * * @param appAddress app address - * @param rmHa resource manager ha + * @param rmHa resource manager ha * @return app address */ public static String getAppAddress(String appAddress, String rmHa) { @@ -636,9 +637,6 @@ public class HadoopUtils implements Closeable { /** * get active resourcemanager - * - * @param rmIds - * @return */ public static String getAcitveRMName(String rmIds) { @@ -669,9 +667,6 @@ public class HadoopUtils implements Closeable { /** * get ResourceManager state - * - * @param url - * @return */ public static String getRMState(String url) { diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 0fbf5f4091..1377335b78 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -58,7 +58,7 @@ fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. -yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s +yarn.application.status.address=http://10.172.33.1:%s/ws/v1/cluster/apps/%s # if custom you resourcemanager port ,you need to replace 8088 else default value. resource.manager.httpaddress.port=8088 From faa111674fb77bc8e4486e62a578b129e403012b Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Mon, 12 Apr 2021 21:41:30 +0800 Subject: [PATCH 18/18] Add issue robot automatic reply and Translation (#5258) * Add issue robot automatic reply and Translation * add license Co-authored-by: chenxingchun <438044805@qq.com> --- .github/workflows/issue_robot.yml | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/issue_robot.yml diff --git a/.github/workflows/issue_robot.yml b/.github/workflows/issue_robot.yml new file mode 100644 index 0000000000..cf940c15b5 --- /dev/null +++ b/.github/workflows/issue_robot.yml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: issue-robot + +on: + issues: + types: [opened] + +jobs: + comment: + runs-on: ubuntu-latest + steps: + - uses: ben-z/actions-comment-on-issue@1.0.2 + with: + message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order to let us know your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email，Mail subscription steps reference https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html ,then send questions to dev@dolphinscheduler.apache.org." + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + autoTranslate: + runs-on: ubuntu-latest + steps: + - uses: actions-cool/translation-helper@v1.1.1 + + automate-issues-labels: + runs-on: ubuntu-latest + steps: + - name: initial labeling + uses: andymckay/labeler@master + with: + add-labels: "Waiting for reply" + ignore-if-labeled: true