From 7af20ca3afe858f29abdd9ad9cb5013d8fd33d65 Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Sat, 12 Sep 2020 23:42:40 +0800 Subject: [PATCH 1/6] [Bug-3713][HadoopUtils] catfile method Stream not closed (#3715) * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * fix bug Delete invalid field: executorcores Modify verification prompt * dag add close button * reset last version * reset last version * dag add close buttion dag add close buttion * update CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * updae CLICK_SAVE_WORKFLOW_BUTTON xpath * Update CreateWorkflowLocator.java modify submit workflow button * Update CreateWorkflowLocator.java * Update CreateWorkflowLocator.java modify CLICK_ADD_BUTTON * Update CreateWorkflowLocator.java delete print * Update CreateWorkflowLocator.java 1 * Update CreateWorkflowLocator.java 1 * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dag.vue * Update dag.vue * Update dag.vue * Update CreateWorkflowLocator.java * Revert "Setting '-XX:+DisableExplicitGC ' causes netty memory leaks" This reverts commit 3a2cba7a * Setting '-XX:+DisableExplicitGC ' causes netty memory leaks in addition update '- XX: largepagesizeinbytes = 128M' to '- XX: largepagesizeinbytes = 10M' * Update dolphinscheduler-daemon.sh * catfile method Stream not closed * catfile method Stream not closed Co-authored-by: dailidong Co-authored-by: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> --- .../common/utils/HadoopUtils.java | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 68d03506c6..fe9ac932c1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -14,26 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.commons.io.IOUtils; +import static org.apache.dolphinscheduler.common.Constants.RESOURCE_UPLOAD_PATH; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.enums.ResourceType; + +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.client.cli.RMAdminCLI; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; import java.nio.file.Files; import java.security.PrivilegedExceptionAction; import java.util.Collections; @@ -43,7 +48,13 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.dolphinscheduler.common.Constants.RESOURCE_UPLOAD_PATH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; /** * hadoop utils @@ -102,7 +113,6 @@ public class HadoopUtils implements Closeable { } } - /** * init hadoop configuration */ @@ -167,7 +177,6 @@ public class HadoopUtils implements Closeable { fs = FileSystem.get(configuration); } - } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -232,11 +241,11 @@ public class HadoopUtils implements Closeable { return new byte[0]; } - FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath)); - return IOUtils.toByteArray(fsDataInputStream); + try (FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath))) { + return IOUtils.toByteArray(fsDataInputStream); + } } - /** * cat file on hdfs * @@ -527,7 +536,6 @@ public class HadoopUtils implements Closeable { return String.format("%s/udfs", getHdfsTenantDir(tenantCode)); } - /** * get hdfs file name * @@ -579,7 +587,6 @@ public class HadoopUtils implements Closeable { return String.format("%s/%s", getHdfsDataBasePath(), tenantCode); } - /** * getAppAddress * @@ -610,7 +617,6 @@ public class HadoopUtils implements Closeable { return start + activeRM + end; } - @Override public void close() throws IOException { if (fs != null) { @@ -623,7 +629,6 @@ public class HadoopUtils implements Closeable { } } - /** * yarn ha admin utils */ @@ -669,7 +674,6 @@ public class HadoopUtils implements Closeable { return null; } - /** * get ResourceManager state * @@ -694,4 +698,5 @@ public class HadoopUtils implements Closeable { } } + } From 94247d84320ef21c9c02ea55edafede112950f75 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Sun, 13 Sep 2020 23:52:19 +0800 Subject: [PATCH 2/6] [Bug-3722][ui]fix cannot change password on dev branch. (#3724) #3722 fix cannot change password on dev branch. --- .../js/conf/home/pages/user/pages/password/_source/info.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue index af296c9f28..c2d97be2f5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/user/pages/password/_source/info.vue @@ -85,7 +85,8 @@ userPassword: this.userPassword, tenantId: this.userInfo.tenantId, email: this.userInfo.email, - phone: this.userInfo.phone + phone: this.userInfo.phone, + state: this.userinfo.state } this.spinnerLoading = true this.updateUser(param).then(res => { From 3a2663eeb7579d1574af0c7ed051dbcbd81efa10 Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Sun, 13 Sep 2020 23:55:25 +0800 Subject: [PATCH 3/6] =?UTF-8?q?[Improvement][DAO]=20druid=20The=20lower=20?= =?UTF-8?q?version=20is=20not=20compatible=20with=20LocalDateTime(DRUID=20?= =?UTF-8?q?=E4=BD=8E=E7=89=88=E6=9C=AC=E4=B8=8D=E6=94=AF=E6=8C=81LocalDate?= =?UTF-8?q?Time)=20(#3716)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix mybatis 3.51版本的LocalDateTime映射报SQLFeatureNotSupportedException * fix druid-1.1.14.jar->1.1.22 --- pom.xml | 2 +- tools/dependencies/known-dependencies.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 0321c1a24c..65cbe62bbb 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 3.2.0 2.0.1 5.0.5 - 1.1.14 + 1.1.22 1.4.200 1.6 1.1.1 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 0074dcd9fb..4a7f1662f0 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -43,7 +43,7 @@ datanucleus-api-jdo-4.2.1.jar datanucleus-core-4.1.6.jar datanucleus-rdbms-4.1.7.jar derby-10.14.2.0.jar -druid-1.1.14.jar +druid-1.1.22.jar gson-2.8.5.jar guava-20.0.jar guice-3.0.jar From 565b8d3b66cbc035d263ef5695a6c01e8d2eac9b Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Mon, 14 Sep 2020 10:39:37 +0800 Subject: [PATCH 4/6] =?UTF-8?q?[Improvement-3690][common]=20Get=20the=20na?= =?UTF-8?q?tive=20IP=20policy=20problem=20(=E8=8E=B7=E5=8F=96=E6=9C=AC?= =?UTF-8?q?=E6=9C=BAip=E7=AD=96=E7=95=A5=E9=97=AE=E9=A2=98=20)=20(#3695)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Get the Intranet IP * Get the Intranet IP * fix code smell * fix code smell * fix code smell * fix code smell * support ipv6 --x:--x:--x:--x:--x:--x:--x:--x * Update dolphinscheduler-common/src/main/resources/common.properties Co-authored-by: Yichao Yang <1048262223@qq.com> Co-authored-by: Yichao Yang <1048262223@qq.com> --- .../dolphinscheduler/common/Constants.java | 5 + .../common/utils/NetUtils.java | 121 ++++++++++++++---- .../src/main/resources/common.properties | 3 + 3 files changed, 103 insertions(+), 26 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 3b12748888..ba8b0c4921 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -1000,4 +1000,9 @@ public final class Constants { public static final String DATASOURCE_ENCRYPTION_SALT_DEFAULT = "!@#$%^&*"; public static final String DATASOURCE_ENCRYPTION_ENABLE = "datasource.encryption.enable"; public static final String DATASOURCE_ENCRYPTION_SALT = "datasource.encryption.salt"; + + /** + * Network IP gets priority, default inner outer + */ + public static final String NETWORK_PRIORITY_STRATEGY = "dolphin.scheduler.network.priority.strategy"; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index b001825ce1..ddb29730b7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -21,6 +21,8 @@ import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PRE import static java.util.Collections.emptyList; +import org.apache.dolphinscheduler.common.Constants; + import java.io.IOException; import java.net.Inet6Address; import java.net.InetAddress; @@ -42,22 +44,20 @@ import org.slf4j.LoggerFactory; */ public class NetUtils { - private NetUtils() { - throw new UnsupportedOperationException("Construct NetUtils"); - } - - private static Logger logger = LoggerFactory.getLogger(NetUtils.class); - private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); - - private static String ANY_HOST_VALUE = "0.0.0.0"; - - private static String LOCAL_HOST_VALUE = "127.0.0.1"; - + private static final String NETWORK_PRIORITY_DEFAULT = "default"; + private static final String NETWORK_PRIORITY_INNER = "inner"; + private static final String NETWORK_PRIORITY_OUTER = "outer"; + private static final Logger logger = LoggerFactory.getLogger(NetUtils.class); + private static final String ANY_HOST_VALUE = "0.0.0.0"; + private static final String LOCAL_HOST_VALUE = "127.0.0.1"; private static InetAddress LOCAL_ADDRESS = null; - private static volatile String HOST_ADDRESS; + private NetUtils() { + throw new UnsupportedOperationException("Construct NetUtils"); + } + public static String getHost() { if (HOST_ADDRESS != null) { return HOST_ADDRESS; @@ -87,24 +87,27 @@ public class NetUtils { if (null != LOCAL_ADDRESS) { return LOCAL_ADDRESS; } + InetAddress localAddress = null; - NetworkInterface networkInterface = findNetworkInterface(); - Enumeration addresses = networkInterface.getInetAddresses(); - while (addresses.hasMoreElements()) { - Optional addressOp = toValidAddress(addresses.nextElement()); - if (addressOp.isPresent()) { - try { - if (addressOp.get().isReachable(100)) { - LOCAL_ADDRESS = addressOp.get(); - return LOCAL_ADDRESS; + try { + NetworkInterface networkInterface = findNetworkInterface(); + if (networkInterface != null) { + Enumeration addresses = networkInterface.getInetAddresses(); + while (addresses.hasMoreElements()) { + Optional addressOp = toValidAddress(addresses.nextElement()); + if (addressOp.isPresent()) { + try { + if (addressOp.get().isReachable(100)) { + LOCAL_ADDRESS = addressOp.get(); + return LOCAL_ADDRESS; + } + } catch (IOException e) { + logger.warn("test address id reachable io exception", e); + } } - } catch (IOException e) { - logger.warn("test address id reachable io exception", e); } } - } - try { localAddress = InetAddress.getLocalHost(); } catch (UnknownHostException e) { logger.warn("InetAddress get LocalHost exception", e); @@ -190,7 +193,7 @@ public class NetUtils { if (null != result) { return result; } - return validNetworkInterfaces.get(0); + return findAddress(validNetworkInterfaces); } /** @@ -227,4 +230,70 @@ public class NetUtils { String preferredNetworkInterface = System.getProperty(DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE); return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface); } + + private static NetworkInterface findAddress(List validNetworkInterfaces) { + if (validNetworkInterfaces.isEmpty()) { + return null; + } + String networkPriority = PropertyUtils.getString(Constants.NETWORK_PRIORITY_STRATEGY, NETWORK_PRIORITY_DEFAULT); + if (NETWORK_PRIORITY_DEFAULT.equalsIgnoreCase(networkPriority)) { + return findAddressByDefaultPolicy(validNetworkInterfaces); + } else if (NETWORK_PRIORITY_INNER.equalsIgnoreCase(networkPriority)) { + return findInnerAddress(validNetworkInterfaces); + } else if (NETWORK_PRIORITY_OUTER.equalsIgnoreCase(networkPriority)) { + return findOuterAddress(validNetworkInterfaces); + } else { + logger.error("There is no matching network card acquisition policy!"); + return null; + } + } + + private static NetworkInterface findAddressByDefaultPolicy(List validNetworkInterfaces) { + NetworkInterface networkInterface; + networkInterface = findInnerAddress(validNetworkInterfaces); + if (networkInterface == null) { + networkInterface = findOuterAddress(validNetworkInterfaces); + if (networkInterface == null) { + networkInterface = validNetworkInterfaces.get(0); + } + } + return networkInterface; + } + + /** + * Get the Intranet IP + * + * @return If no {@link NetworkInterface} is available , return null + */ + private static NetworkInterface findInnerAddress(List validNetworkInterfaces) { + + NetworkInterface networkInterface = null; + for (NetworkInterface ni : validNetworkInterfaces) { + Enumeration address = ni.getInetAddresses(); + while (address.hasMoreElements()) { + InetAddress ip = address.nextElement(); + if (ip.isSiteLocalAddress() + && !ip.isLoopbackAddress()) { + networkInterface = ni; + } + } + } + return networkInterface; + } + + private static NetworkInterface findOuterAddress(List validNetworkInterfaces) { + NetworkInterface networkInterface = null; + for (NetworkInterface ni : validNetworkInterfaces) { + Enumeration address = ni.getInetAddresses(); + while (address.hasMoreElements()) { + InetAddress ip = address.nextElement(); + if (!ip.isSiteLocalAddress() + && !ip.isLoopbackAddress()) { + networkInterface = ni; + } + } + } + return networkInterface; + } + } diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index a75f964fa2..b8c21c853b 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -72,3 +72,6 @@ kerberos.expire.time=2 # datasource encryption salt datasource.encryption.enable=false datasource.encryption.salt=!@#$%^&* + +# Network IP gets priority, default inner outer +#dolphin.scheduler.network.priority.strategy=default From fad28527685453082cc2950da835d477c9ec5a54 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Mon, 14 Sep 2020 15:59:56 +0800 Subject: [PATCH 5/6] [Improvement][server] WATERDROP task plug-in optimization in switch case code cleaning. (#3652) * WATERDROP switch case code checkstyle. * add TaskManagerTest. * TaskManagerTest checkstyle. * TaskManagerTest checkstyle. * TaskManagerTest add pom maven-surefire * TaskManagerTest update. --- .../common/utils/TaskParametersUtils.java | 3 +- .../server/worker/task/TaskManager.java | 74 ++++++------- .../server/worker/task/TaskManagerTest.java | 103 ++++++++++++++++++ pom.xml | 1 + 4 files changed, 141 insertions(+), 40 deletions(-) create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index 6099a0d49d..2b40b079c6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -59,9 +59,8 @@ public class TaskParametersUtils { switch (EnumUtils.getEnum(TaskType.class, taskType)) { case SUB_PROCESS: return JSONUtils.parseObject(parameter, SubProcessParameters.class); - case WATERDROP: - return JSONUtils.parseObject(parameter, ShellParameters.class); case SHELL: + case WATERDROP: return JSONUtils.parseObject(parameter, ShellParameters.class); case PROCEDURE: return JSONUtils.parseObject(parameter, ProcedureParameters.class); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java index f98d451350..34eea9dade 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/TaskManager.java @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.worker.task; +package org.apache.dolphinscheduler.server.worker.task; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.utils.EnumUtils; @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.server.worker.task.shell.ShellTask; import org.apache.dolphinscheduler.server.worker.task.spark.SparkTask; import org.apache.dolphinscheduler.server.worker.task.sql.SqlTask; import org.apache.dolphinscheduler.server.worker.task.sqoop.SqoopTask; + import org.slf4j.Logger; /** @@ -37,42 +38,39 @@ import org.slf4j.Logger; */ public class TaskManager { - /** - * create new task - * @param taskExecutionContext taskExecutionContext - * @param logger logger - * @return AbstractTask - * @throws IllegalArgumentException illegal argument exception - */ - public static AbstractTask newTask(TaskExecutionContext taskExecutionContext, - Logger logger) - throws IllegalArgumentException { - switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) { - case SHELL: - return new ShellTask(taskExecutionContext, logger); - case WATERDROP: - return new ShellTask(taskExecutionContext, logger); - case PROCEDURE: - return new ProcedureTask(taskExecutionContext, logger); - case SQL: - return new SqlTask(taskExecutionContext, logger); - case MR: - return new MapReduceTask(taskExecutionContext, logger); - case SPARK: - return new SparkTask(taskExecutionContext, logger); - case FLINK: - return new FlinkTask(taskExecutionContext, logger); - case PYTHON: - return new PythonTask(taskExecutionContext, logger); - case HTTP: - return new HttpTask(taskExecutionContext, logger); - case DATAX: - return new DataxTask(taskExecutionContext, logger); - case SQOOP: - return new SqoopTask(taskExecutionContext, logger); - default: - logger.error("unsupport task type: {}", taskExecutionContext.getTaskType()); - throw new IllegalArgumentException("not support task type"); + /** + * create new task + * @param taskExecutionContext taskExecutionContext + * @param logger logger + * @return AbstractTask + * @throws IllegalArgumentException illegal argument exception + */ + public static AbstractTask newTask(TaskExecutionContext taskExecutionContext, Logger logger) throws IllegalArgumentException { + switch (EnumUtils.getEnum(TaskType.class,taskExecutionContext.getTaskType())) { + case SHELL: + case WATERDROP: + return new ShellTask(taskExecutionContext, logger); + case PROCEDURE: + return new ProcedureTask(taskExecutionContext, logger); + case SQL: + return new SqlTask(taskExecutionContext, logger); + case MR: + return new MapReduceTask(taskExecutionContext, logger); + case SPARK: + return new SparkTask(taskExecutionContext, logger); + case FLINK: + return new FlinkTask(taskExecutionContext, logger); + case PYTHON: + return new PythonTask(taskExecutionContext, logger); + case HTTP: + return new HttpTask(taskExecutionContext, logger); + case DATAX: + return new DataxTask(taskExecutionContext, logger); + case SQOOP: + return new SqoopTask(taskExecutionContext, logger); + default: + logger.error("unsupport task type: {}", taskExecutionContext.getTaskType()); + throw new IllegalArgumentException("not support task type"); + } } - } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java new file mode 100644 index 0000000000..058270e1e1 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/TaskManagerTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.task; + +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import java.util.Date; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({SpringApplicationContext.class}) +public class TaskManagerTest { + + private TaskExecutionContext taskExecutionContext; + + private Logger taskLogger; + + private TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager; + + @Before + public void before() { + // init task execution context, logger + taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setProcessId(12345); + taskExecutionContext.setProcessDefineId(1); + taskExecutionContext.setProcessInstanceId(1); + taskExecutionContext.setTaskInstanceId(1); + taskExecutionContext.setTaskType(""); + taskExecutionContext.setFirstSubmitTime(new Date()); + taskExecutionContext.setDelayTime(0); + taskExecutionContext.setLogPath("/tmp/test.log"); + taskExecutionContext.setHost("localhost"); + taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); + + taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId( + LoggerUtils.TASK_LOGGER_INFO_PREFIX, + taskExecutionContext.getProcessDefineId(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId() + )); + + taskExecutionContextCacheManager = new TaskExecutionContextCacheManagerImpl(); + taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); + + PowerMockito.mockStatic(SpringApplicationContext.class); + PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) + .thenReturn(taskExecutionContextCacheManager); + } + + @Test + public void testNewTask() { + + taskExecutionContext.setTaskType("SHELL"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("WATERDROP"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("HTTP"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("MR"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("SPARK"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("FLINK"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("PYTHON"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("DATAX"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + taskExecutionContext.setTaskType("SQOOP"); + Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + //taskExecutionContext.setTaskType(null); + //Assert.assertNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + //taskExecutionContext.setTaskType("XXX"); + //Assert.assertNotNull(TaskManager.newTask(taskExecutionContext,taskLogger)); + } +} diff --git a/pom.xml b/pom.xml index 65cbe62bbb..207518c936 100644 --- a/pom.xml +++ b/pom.xml @@ -847,6 +847,7 @@ **/server/worker/task/sqoop/SqoopTaskTest.java + **/server/worker/task/TaskManagerTest.java **/server/worker/EnvFileTest.java **/server/worker/runner/TaskExecuteThreadTest.java **/service/quartz/cron/CronUtilsTest.java From 3942740941533721694ceef2177c5363b0d0f200 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Mon, 14 Sep 2020 18:56:47 +0800 Subject: [PATCH 6/6] js mailbox verification fix. (#3721) --- .../pages/projects/pages/definition/pages/list/_source/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js index 308af45a48..877f349803 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/util.js @@ -37,7 +37,7 @@ const warningTypeList = [ ] const isEmial = (val) => { - let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$/ // eslint-disable-line + let regEmail = /^([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+@([a-zA-Z0-9]+[_|\-|\.]?)*[a-zA-Z0-9]+\.[a-zA-Z]{2,}$/ // eslint-disable-line return regEmail.test(val) }