From 0ba120e8ab80d4687685837f6aa52460e7201813 Mon Sep 17 00:00:00 2001 From: mrproliu <741550557@qq.com> Date: Tue, 16 Mar 2021 21:56:05 +0800 Subject: [PATCH 01/13] Fix shell task ${setValue(key,value)} is not working (#5067) Co-authored-by: dalekliuhan --- .../dolphinscheduler/server/worker/task/shell/ShellTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index fb0a76cff2..85f8ea094b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -106,6 +106,7 @@ public class ShellTask extends AbstractTask { setAppIds(commandExecuteResult.getAppIds()); setProcessId(commandExecuteResult.getProcessId()); setResult(shellCommandExecutor.getTaskResultString()); + setVarPool(shellCommandExecutor.getVarPool()); } catch (Exception e) { logger.error("shell task error", e); setExitStatusCode(Constants.EXIT_CODE_FAILURE); From 1cd62b4a5e6d5e8ac3adf8b7e2ffd61a8e87f71b Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Wed, 17 Mar 2021 14:38:04 +0800 Subject: [PATCH 02/13] [Improvement][Datax] Re-assign DATAX_HOME to /opt/soft/datax instead of /opt/soft/datax/bin/datax.py (#5019) --- .../DOLPHIN/1.3.0/configuration/dolphin-env.xml | 3 ++- .../DOLPHIN/1.3.3/configuration/dolphin-env.xml | 3 ++- docker/build/README.md | 2 +- docker/build/README_zh_CN.md | 2 +- .../dolphinscheduler/env/dolphinscheduler_env.sh.tpl | 2 +- docker/build/startup-init-conf.sh | 2 +- docker/docker-swarm/docker-compose.yml | 2 +- docker/docker-swarm/docker-stack.yml | 2 +- docker/kubernetes/dolphinscheduler/values.yaml | 4 ++-- .../common/task/DataxParametersTest.java | 4 ++-- .../server/worker/task/datax/DataxTask.java | 10 +++++----- script/env/dolphinscheduler_env.sh | 4 ++-- 12 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml index 8e14716d05..18501c6d0a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.0/configuration/dolphin-env.xml @@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2 export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax content false diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml index 8e14716d05..18501c6d0a 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.3/configuration/dolphin-env.xml @@ -112,7 +112,8 @@ export SPARK_HOME2=/opt/soft/spark2 export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink +export FLINK_HOME=/opt/soft/flink +export DATAX_HOME=/opt/soft/datax content false diff --git a/docker/build/README.md b/docker/build/README.md index 4d3ac8767b..5a1b28a715 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -209,7 +209,7 @@ This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/fli **`DATAX_HOME`** -This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax/bin/datax.py`. +This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax`. **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 1dfc69c5ad..f3b8ea723e 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -209,7 +209,7 @@ DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会 **`DATAX_HOME`** -配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax/bin/datax。py`。 +配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax`。 **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl index 4463a3f227..b9b04eb83d 100755 --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl @@ -25,4 +25,4 @@ export HIVE_HOME=$HIVE_HOME export FLINK_HOME=$FLINK_HOME export DATAX_HOME=$DATAX_HOME -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index b16100c405..80baa92d25 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -46,7 +46,7 @@ export PYTHON_HOME=${PYTHON_HOME:-"/usr"} export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} -export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"} +export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax"} # common env export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 5b2cf46b12..8f6a0b85cc 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -181,7 +181,7 @@ services: JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" - DATAX_HOME: "/opt/soft/datax/bin/datax.py" + DATAX_HOME: "/opt/soft/datax" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index aab44a388d..d78e7261e7 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -175,7 +175,7 @@ services: JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" HIVE_HOME: "/opt/soft/hive" FLINK_HOME: "/opt/soft/flink" - DATAX_HOME: "/opt/soft/datax/bin/datax.py" + DATAX_HOME: "/opt/soft/datax" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index ba6b8eb8f8..af7da46235 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -80,8 +80,8 @@ common: - "export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk" - "export HIVE_HOME=/opt/soft/hive" - "export FLINK_HOME=/opt/soft/flink" - - "export DATAX_HOME=/opt/soft/datax/bin/datax.py" - - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$PATH" + - "export DATAX_HOME=/opt/soft/datax" + - "export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" RESOURCE_STORAGE_TYPE: "HDFS" RESOURCE_UPLOAD_PATH: "/dolphinscheduler" diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java index d6e2f69882..dd11aab260 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/DataxParametersTest.java @@ -27,7 +27,7 @@ public class DataxParametersTest { /** * jvm parameters */ - public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" "; + public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; @Test public void testLoadJvmEnv() { @@ -88,6 +88,6 @@ public class DataxParametersTest { public String loadJvmEnvTest(DataxParameters dataXParameters) { int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); - return String.format(JVM_EVN, xms, xmx); + return String.format(JVM_PARAM, xms, xmx); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index caf487947b..8f0b20d17f 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -81,15 +81,15 @@ public class DataxTask extends AbstractTask { /** * jvm parameters */ - public static final String JVM_EVN = " --jvm=\"-Xms%sG -Xmx%sG\" "; + public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" "; /** * python process(datax only supports version 2.7 by default) */ private static final String DATAX_PYTHON = "python2.7"; /** - * datax home path + * datax path */ - private static final String DATAX_HOME_EVN = "${DATAX_HOME}"; + private static final String DATAX_PATH = "${DATAX_HOME}/bin/datax.py"; /** * datax channel count */ @@ -396,7 +396,7 @@ public class DataxTask extends AbstractTask { StringBuilder sbr = new StringBuilder(); sbr.append(DATAX_PYTHON); sbr.append(" "); - sbr.append(DATAX_HOME_EVN); + sbr.append(DATAX_PATH); sbr.append(" "); sbr.append(loadJvmEnv(dataXParameters)); sbr.append(jobConfigFilePath); @@ -424,7 +424,7 @@ public class DataxTask extends AbstractTask { public String loadJvmEnv(DataxParameters dataXParameters) { int xms = dataXParameters.getXms() < 1 ? 1 : dataXParameters.getXms(); int xmx = dataXParameters.getXmx() < 1 ? 1 : dataXParameters.getXmx(); - return String.format(JVM_EVN, xms, xmx); + return String.format(JVM_PARAM, xms, xmx); } /** diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh index 9fb9f55d70..e62f54e396 100755 --- a/script/env/dolphinscheduler_env.sh +++ b/script/env/dolphinscheduler_env.sh @@ -23,6 +23,6 @@ export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive export FLINK_HOME=/opt/soft/flink -export DATAX_HOME=/opt/soft/datax/bin/datax.py +export DATAX_HOME=/opt/soft/datax -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH +export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH From 6216817861c64d9a170686c4c5aa6f3b9b0da110 Mon Sep 17 00:00:00 2001 From: Ts686 <30523983+Ts686@users.noreply.github.com> Date: Wed, 17 Mar 2021 15:28:07 +0800 Subject: [PATCH 03/13] [Improvement-4971][alert|dao|service]Remove redundant PropertyUtils.class (#5020) * [Improvement][alert|dao|service] 1. Remove redundant PropertyUtils.class (#4971) 2. Remove useless import package. * [Improvement][alert|dao|service] 1. Remove redundant PropertyUtils.class (#4971) 2. Remove useless import package. * [Improvement][alert|dao|service] 1. Remove redundant PropertyUtils.class (#4971) 2. Remove useless import package. * [Improvement][alert|dao|service] 1. Remove redundant PropertyUtils.class (#4971) 2. Remove useless import package. --- .../dolphinscheduler/alert/AlertServer.java | 8 +- .../alert/utils/PropertyUtils.java | 262 ------------------ .../alert/plugin/AlertPluginManagerTest.java | 4 +- .../alert/plugin/EmailAlertPluginTest.java | 2 +- .../alert/utils/PropertyUtilsTest.java | 9 +- .../common/utils/PropertyUtils.java | 16 +- .../dolphinscheduler/dao/TaskRecordDao.java | 39 ++- .../datasource/SpringConnectionFactory.java | 77 ++--- .../dao/utils/PropertyUtils.java | 162 ----------- .../service/quartz/QuartzExecutors.java | 5 +- 10 files changed, 95 insertions(+), 489 deletions(-) delete mode 100644 dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index 38281933cd..d0fb66af4b 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -17,16 +17,17 @@ package org.apache.dolphinscheduler.alert; +import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH; import static org.apache.dolphinscheduler.common.Constants.ALERT_RPC_PORT; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; -import org.apache.dolphinscheduler.common.plugin.DolphinPluginLoader; -import org.apache.dolphinscheduler.common.plugin.DolphinPluginManagerConfig; import org.apache.dolphinscheduler.alert.processor.AlertRequestProcessor; import org.apache.dolphinscheduler.alert.runner.AlertSender; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.plugin.DolphinPluginLoader; +import org.apache.dolphinscheduler.common.plugin.DolphinPluginManagerConfig; import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.entity.Alert; @@ -140,6 +141,7 @@ public class AlertServer { * start */ public void start() { + PropertyUtils.loadPropertyFile(ALERT_PROPERTIES_PATH); initPlugin(); initRemoteServer(); logger.info("alert server ready start "); diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java deleted file mode 100644 index 16bcb6bd79..0000000000 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/PropertyUtils.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.alert.utils; - -import static org.apache.dolphinscheduler.alert.utils.Constants.ALERT_PROPERTIES_PATH; - -import org.apache.dolphinscheduler.common.utils.IOUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; -import java.util.regex.PatternSyntaxException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * property utils - * single instance - */ -public class PropertyUtils { - - /** - * logger - */ - private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); - - private static final Properties properties = new Properties(); - - /** - * init properties - */ - private static final PropertyUtils propertyUtils = new PropertyUtils(); - - private PropertyUtils() { - init(); - } - - private void init() { - String[] propertyFiles = new String[]{ALERT_PROPERTIES_PATH}; - for (String fileName : propertyFiles) { - InputStream fis = null; - try { - fis = PropertyUtils.class.getResourceAsStream(fileName); - properties.load(fis); - - } catch (IOException e) { - logger.error(e.getMessage(), e); - if (fis != null) { - IOUtils.closeQuietly(fis); - } - System.exit(1); - } finally { - IOUtils.closeQuietly(fis); - } - } - } - - /** - * get property value - * - * @param key property name - * @return the value - */ - public static String getString(String key) { - if (StringUtils.isEmpty(key)) { - return null; - } - return properties.getProperty(key.trim()); - } - - /** - * get property value - * - * @param key property name - * @param defaultVal default value - * @return property value - */ - public static String getString(String key, String defaultVal) { - String val = properties.getProperty(key.trim()); - return val == null ? defaultVal : val; - } - - /** - * get property value - * - * @param key property name - * @return get property int value , if key == null, then return -1 - */ - public static int getInt(String key) { - - return getInt(key, -1); - } - - /** - * get int value - * - * @param key the key - * @param defaultValue the default value - * @return the value related the key or the default value if the key not existed - */ - public static int getInt(String key, int defaultValue) { - String value = getString(key); - if (value == null) { - return defaultValue; - } - - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - logger.info(e.getMessage(), e); - } - return defaultValue; - } - - /** - * get property value - * - * @param key property name - * @return the boolean result value - */ - public static Boolean getBoolean(String key) { - - if (StringUtils.isEmpty(key)) { - return false; - } - - String value = properties.getProperty(key.trim()); - if (null != value) { - return Boolean.parseBoolean(value); - } - - return false; - } - - /** - * get long value - * - * @param key the key - * @return if the value not existed, return -1, or will return the related value - */ - public static long getLong(String key) { - return getLong(key, -1); - } - - /** - * get long value - * - * @param key the key - * @param defaultVal the default value - * @return the value related the key or the default value if the key not existed - */ - public static long getLong(String key, long defaultVal) { - - String val = getString(key); - if (val == null) { - return defaultVal; - } - - try { - return Long.parseLong(val); - } catch (NumberFormatException e) { - logger.info(e.getMessage(), e); - } - - return defaultVal; - } - - /** - * get double value - * - * @param key the key - * @return if the value not existed, return -1.0, or will return the related value - */ - public static double getDouble(String key) { - return getDouble(key, -1.0); - } - - /** - * get double value - * - * @param key the key - * @param defaultVal the default value - * @return the value related the key or the default value if the key not existed - */ - public static double getDouble(String key, double defaultVal) { - - String val = getString(key); - if (val == null) { - return defaultVal; - } - - try { - return Double.parseDouble(val); - } catch (NumberFormatException e) { - logger.info(e.getMessage(), e); - } - - return defaultVal; - } - - /** - * get array - * - * @param key property name - * @param splitStr separator - * @return the result array - */ - public static String[] getArray(String key, String splitStr) { - String value = getString(key); - if (value == null || StringUtils.isEmpty(splitStr)) { - return null; - } - try { - return value.split(splitStr); - } catch (PatternSyntaxException e) { - logger.info(e.getMessage(), e); - } - return null; - } - - /** - * get enum - * - * @param key the key - * @param type the class type - * @param defaultValue the default value - * @param the generic class type - * @return get enum value - */ - public static > T getEnum(String key, Class type, - T defaultValue) { - String val = getString(key); - if (val == null) { - return defaultValue; - } - - try { - return Enum.valueOf(type, val); - } catch (IllegalArgumentException e) { - logger.info(e.getMessage(), e); - } - - return defaultValue; - } -} diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManagerTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManagerTest.java index dbf43295da..5e33b46a7d 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManagerTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/AlertPluginManagerTest.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.alert.plugin; import org.apache.dolphinscheduler.alert.AlertServer; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.plugin.DolphinPluginLoader; import org.apache.dolphinscheduler.common.plugin.DolphinPluginManagerConfig; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -48,7 +48,7 @@ public class AlertPluginManagerTest { String path = Objects.requireNonNull(DolphinPluginLoader.class.getClassLoader().getResource("")).getPath(); alertPluginManagerConfig.setPlugins(path + "../../../dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml"); if (StringUtils.isNotBlank(PropertyUtils.getString(AlertServer.ALERT_PLUGIN_DIR))) { - alertPluginManagerConfig.setInstalledPluginsDir(org.apache.dolphinscheduler.alert.utils.PropertyUtils.getString(AlertServer.ALERT_PLUGIN_DIR, Constants.ALERT_PLUGIN_PATH).trim()); + alertPluginManagerConfig.setInstalledPluginsDir(PropertyUtils.getString(AlertServer.ALERT_PLUGIN_DIR, Constants.ALERT_PLUGIN_PATH).trim()); } if (StringUtils.isNotBlank(PropertyUtils.getString(AlertServer.MAVEN_LOCAL_REPOSITORY))) { diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java index e02c979d2c..958c1b44b7 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/plugin/EmailAlertPluginTest.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.alert.plugin; import org.apache.dolphinscheduler.alert.AlertServer; import org.apache.dolphinscheduler.alert.runner.AlertSender; import org.apache.dolphinscheduler.alert.utils.Constants; -import org.apache.dolphinscheduler.alert.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.plugin.DolphinPluginLoader; import org.apache.dolphinscheduler.common.plugin.DolphinPluginManagerConfig; diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java index 68f90b588d..5d5d3d9e94 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/PropertyUtilsTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.junit.Test; @@ -119,15 +120,15 @@ public class PropertyUtilsTest { public void testGetDouble() { //Expected 3.0 - double result = PropertyUtils.getDouble("test.server.factor"); + double result = PropertyUtils.getDouble("test.server.factor", 3.0); assertEquals(3.0, result, 0); //If key is null, then return -1.0 - result = PropertyUtils.getDouble(null); + result = PropertyUtils.getDouble(null, -1.0); assertEquals(-1.0, result, 0); //If key is undefine in alert.properties, then return -1 - result = PropertyUtils.getDouble("abc"); + result = PropertyUtils.getDouble("abc", -1.0); assertEquals(-1.0, result, 0); //If key is undefine in alert.properties, and there is a defaultval, then return defaultval @@ -135,7 +136,7 @@ public class PropertyUtilsTest { assertEquals(5.0, result, 0); //If the value can not parse to double ,it will log the error and return -1.0 - result = PropertyUtils.getDouble("test.server.testnumber"); + result = PropertyUtils.getDouble("test.server.testnumber", -1.0); assertEquals(-1.0, result, 0); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 0ca80b1231..199655e254 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -22,8 +22,6 @@ import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PAT import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.commons.io.IOUtils; - import java.io.IOException; import java.io.InputStream; import java.util.HashMap; @@ -51,7 +49,13 @@ public class PropertyUtils { } static { - String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; + loadPropertyFile(COMMON_PROPERTIES_PATH); + } + + /** + * init properties + */ + public static synchronized void loadPropertyFile(String... propertyFiles) { for (String fileName : propertyFiles) { InputStream fis = null; try { @@ -196,7 +200,7 @@ public class PropertyUtils { * @param defaultVal default value * @return property value */ - public double getDouble(String key, double defaultVal) { + public static double getDouble(String key, double defaultVal) { String val = getString(key); return val == null ? defaultVal : Double.parseDouble(val); } @@ -229,8 +233,8 @@ public class PropertyUtils { * @param T * @return get enum value */ - public > T getEnum(String key, Class type, - T defaultValue) { + public static > T getEnum(String key, Class type, + T defaultValue) { String val = getString(key); return val == null ? defaultValue : Enum.valueOf(type, val); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java index 1592e607f9..af8d9af386 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/TaskRecordDao.java @@ -14,27 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; +import static org.apache.dolphinscheduler.common.Constants.DATASOURCE_PROPERTIES; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TaskRecordStatus; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.TaskRecord; -import org.apache.dolphinscheduler.dao.utils.PropertyUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.sql.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * task record dao @@ -44,12 +48,17 @@ public class TaskRecordDao { private static Logger logger = LoggerFactory.getLogger(TaskRecordDao.class.getName()); + static { + PropertyUtils.loadPropertyFile(DATASOURCE_PROPERTIES); + } + /** - * get task record flag + * get task record flag + * * @return whether startup taskrecord */ - public static boolean getTaskRecordFlag(){ - return PropertyUtils.getBoolean(Constants.TASK_RECORD_FLAG,false); + public static boolean getTaskRecordFlag() { + return PropertyUtils.getBoolean(Constants.TASK_RECORD_FLAG, false); } /** @@ -132,7 +141,7 @@ public class TaskRecordDao { * count task record * * @param filterMap filterMap - * @param table table + * @param table table * @return task record count */ public static int countTaskRecord(Map filterMap, String table) { @@ -150,13 +159,13 @@ public class TaskRecordDao { sql += getWhereString(filterMap); pstmt = conn.prepareStatement(sql); rs = pstmt.executeQuery(); - while (rs.next()){ + while (rs.next()) { count = rs.getInt("count"); break; } } catch (SQLException e) { logger.error("Exception ", e); - }finally { + } finally { ConnectionUtils.releaseResource(rs, pstmt, conn); } return count; @@ -166,7 +175,7 @@ public class TaskRecordDao { * query task record by filter map paging * * @param filterMap filterMap - * @param table table + * @param table table * @return task record list */ public static List queryAllTaskRecord(Map filterMap, String table) { @@ -241,7 +250,7 @@ public class TaskRecordDao { } } catch (SQLException e) { logger.error("Exception ", e); - }finally { + } finally { ConnectionUtils.releaseResource(rs, pstmt, conn); } return recordList; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java index 5dcdc75ab5..5a0e6353d0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java @@ -14,23 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.datasource; -import com.alibaba.druid.pool.DruidDataSource; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.core.MybatisConfiguration; -import com.baomidou.mybatisplus.core.config.GlobalConfig; -import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; -import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; +import static org.apache.dolphinscheduler.common.Constants.DATASOURCE_PROPERTIES; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.dao.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; + import org.apache.ibatis.mapping.DatabaseIdProvider; import org.apache.ibatis.mapping.VendorDatabaseIdProvider; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.type.JdbcType; + +import java.util.Properties; + import org.mybatis.spring.SqlSessionTemplate; import org.mybatis.spring.annotation.MapperScan; import org.slf4j.Logger; @@ -41,7 +40,12 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.core.io.support.ResourcePatternResolver; import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import java.util.Properties; +import com.alibaba.druid.pool.DruidDataSource; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.config.GlobalConfig; +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; /** @@ -53,9 +57,13 @@ public class SpringConnectionFactory { private static final Logger logger = LoggerFactory.getLogger(SpringConnectionFactory.class); + static { + PropertyUtils.loadPropertyFile(DATASOURCE_PROPERTIES); + } /** - * pagination interceptor + * pagination interceptor + * * @return pagination interceptor */ @Bean @@ -65,9 +73,10 @@ public class SpringConnectionFactory { /** * get the data source + * * @return druid dataSource */ - @Bean(destroyMethod="") + @Bean(destroyMethod = "") public DruidDataSource dataSource() { DruidDataSource druidDataSource = new DruidDataSource(); @@ -76,30 +85,31 @@ public class SpringConnectionFactory { druidDataSource.setUrl(PropertyUtils.getString(Constants.SPRING_DATASOURCE_URL)); druidDataSource.setUsername(PropertyUtils.getString(Constants.SPRING_DATASOURCE_USERNAME)); druidDataSource.setPassword(PropertyUtils.getString(Constants.SPRING_DATASOURCE_PASSWORD)); - druidDataSource.setValidationQuery(PropertyUtils.getString(Constants.SPRING_DATASOURCE_VALIDATION_QUERY,"SELECT 1")); - - druidDataSource.setPoolPreparedStatements(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_POOL_PREPARED_STATEMENTS,true)); - druidDataSource.setTestWhileIdle(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_WHILE_IDLE,true)); - druidDataSource.setTestOnBorrow(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_ON_BORROW,true)); - druidDataSource.setTestOnReturn(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_ON_RETURN,true)); - druidDataSource.setKeepAlive(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_KEEP_ALIVE,true)); - - druidDataSource.setMinIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE,5)); - druidDataSource.setMaxActive(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE,50)); - druidDataSource.setMaxWait(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_WAIT,60000)); - druidDataSource.setMaxPoolPreparedStatementPerConnectionSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_POOL_PREPARED_STATEMENT_PER_CONNECTION_SIZE,20)); - druidDataSource.setInitialSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_INITIAL_SIZE,5)); - druidDataSource.setTimeBetweenEvictionRunsMillis(PropertyUtils.getLong(Constants.SPRING_DATASOURCE_TIME_BETWEEN_EVICTION_RUNS_MILLIS,60000)); - druidDataSource.setTimeBetweenConnectErrorMillis(PropertyUtils.getLong(Constants.SPRING_DATASOURCE_TIME_BETWEEN_CONNECT_ERROR_MILLIS,60000)); - druidDataSource.setMinEvictableIdleTimeMillis(PropertyUtils.getLong(Constants.SPRING_DATASOURCE_MIN_EVICTABLE_IDLE_TIME_MILLIS,300000)); - druidDataSource.setValidationQueryTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_VALIDATION_QUERY_TIMEOUT,3)); + druidDataSource.setValidationQuery(PropertyUtils.getString(Constants.SPRING_DATASOURCE_VALIDATION_QUERY, "SELECT 1")); + + druidDataSource.setPoolPreparedStatements(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_POOL_PREPARED_STATEMENTS, true)); + druidDataSource.setTestWhileIdle(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_WHILE_IDLE, true)); + druidDataSource.setTestOnBorrow(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_ON_BORROW, true)); + druidDataSource.setTestOnReturn(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_TEST_ON_RETURN, true)); + druidDataSource.setKeepAlive(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_KEEP_ALIVE, true)); + + druidDataSource.setMinIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE, 5)); + druidDataSource.setMaxActive(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50)); + druidDataSource.setMaxWait(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_WAIT, 60000)); + druidDataSource.setMaxPoolPreparedStatementPerConnectionSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_POOL_PREPARED_STATEMENT_PER_CONNECTION_SIZE, 20)); + druidDataSource.setInitialSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_INITIAL_SIZE, 5)); + druidDataSource.setTimeBetweenEvictionRunsMillis(PropertyUtils.getLong(Constants.SPRING_DATASOURCE_TIME_BETWEEN_EVICTION_RUNS_MILLIS, 60000)); + druidDataSource.setTimeBetweenConnectErrorMillis(PropertyUtils.getLong(Constants.SPRING_DATASOURCE_TIME_BETWEEN_CONNECT_ERROR_MILLIS, 60000)); + druidDataSource.setMinEvictableIdleTimeMillis(PropertyUtils.getLong(Constants.SPRING_DATASOURCE_MIN_EVICTABLE_IDLE_TIME_MILLIS, 300000)); + druidDataSource.setValidationQueryTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_VALIDATION_QUERY_TIMEOUT, 3)); //auto commit - druidDataSource.setDefaultAutoCommit(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_DEFAULT_AUTO_COMMIT,true)); + druidDataSource.setDefaultAutoCommit(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_DEFAULT_AUTO_COMMIT, true)); return druidDataSource; } /** * * get transaction manager + * * @return DataSourceTransactionManager */ @Bean @@ -109,6 +119,7 @@ public class SpringConnectionFactory { /** * * get sql session factory + * * @return sqlSessionFactory * @throws Exception sqlSessionFactory exception */ @@ -139,16 +150,16 @@ public class SpringConnectionFactory { /** * get sql session + * * @return SqlSession - * @throws Exception */ @Bean - public SqlSession sqlSession() throws Exception{ + public SqlSession sqlSession() throws Exception { return new SqlSessionTemplate(sqlSessionFactory()); } @Bean - public DatabaseIdProvider databaseIdProvider(){ + public DatabaseIdProvider databaseIdProvider() { DatabaseIdProvider databaseIdProvider = new VendorDatabaseIdProvider(); Properties properties = new Properties(); properties.setProperty("MySQL", "mysql"); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java deleted file mode 100644 index 47cfadbf9a..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.utils; - -import org.apache.dolphinscheduler.common.Constants; -import com.baomidou.mybatisplus.core.toolkit.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - - -/** - * property utils - * single instance - */ -public class PropertyUtils { - - /** - * logger - */ - private static final Logger logger = LoggerFactory.getLogger(PropertyUtils.class); - - private static final Properties properties = new Properties(); - - private static final PropertyUtils propertyUtils = new PropertyUtils(); - - private PropertyUtils(){ - init(); - } - - /** - * init - */ - private void init(){ - String[] propertyFiles = new String[]{Constants.DATASOURCE_PROPERTIES}; - for (String fileName : propertyFiles) { - InputStream fis = null; - try { - fis = PropertyUtils.class.getResourceAsStream(fileName); - properties.load(fis); - - } catch (IOException e) { - logger.error(e.getMessage(), e); - if (fis != null) { - IOUtils.closeQuietly(fis); - } - System.exit(1); - } finally { - IOUtils.closeQuietly(fis); - } - } - } - - /** - * get property value - * @param key property name - * @return get string value - */ - public static String getString(String key) { - return properties.getProperty(key); - } - - /** - * get property value - * - * @param key property name - * @param defaultVal default value - * @return property value - */ - public static String getString(String key, String defaultVal) { - String val = properties.getProperty(key.trim()); - return val == null ? defaultVal : val; - } - - /** - * get property value - * @param key property name - * @return get property int value , if key == null, then return -1 - */ - public static int getInt(String key) { - return getInt(key, -1); - } - - /** - * get property value - * @param key key - * @param defaultValue defaultValue - * @return get property int value，if key == null ，then return defaultValue - */ - public static int getInt(String key, int defaultValue) { - String value = getString(key); - if (value == null) { - return defaultValue; - } - - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - logger.info(e.getMessage(),e); - } - return defaultValue; - } - - /** - * get property value - * - * @param key property name - * @return property value - */ - public static Boolean getBoolean(String key) { - String value = properties.getProperty(key.trim()); - if(null != value){ - return Boolean.parseBoolean(value); - } - - return false; - } - - /** - * get property value - * - * @param key property name - * @param defaultValue default value - * @return property value - */ - public static Boolean getBoolean(String key, boolean defaultValue) { - String value = properties.getProperty(key.trim()); - if(null != value){ - return Boolean.parseBoolean(value); - } - - return defaultValue; - } - - /** - * get property long value - * @param key key - * @param defaultVal default value - * @return property value - */ - public static long getLong(String key, long defaultVal) { - String val = getString(key); - return val == null ? defaultVal : Long.parseLong(val); - } -} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 96209af93c..7f53f8ea93 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.service.quartz; +import static org.apache.dolphinscheduler.common.Constants.DATASOURCE_PROPERTIES; import static org.apache.dolphinscheduler.common.Constants.ORG_POSTGRESQL_DRIVER; import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_DATASOURCE_MYDS_CONNECTIONPROVIDER_CLASS; import static org.apache.dolphinscheduler.common.Constants.ORG_QUARTZ_JOBSTORE_ACQUIRETRIGGERSWITHINLOCK; @@ -60,6 +61,7 @@ import static org.quartz.JobBuilder.newJob; import static org.quartz.TriggerBuilder.newTrigger; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.service.exceptions.ServiceException; @@ -147,10 +149,11 @@ public class QuartzExecutors { */ private void init() { try { + PropertyUtils.loadPropertyFile(DATASOURCE_PROPERTIES); StdSchedulerFactory schedulerFactory = new StdSchedulerFactory(); Properties properties = new Properties(); - String dataSourceDriverClass = org.apache.dolphinscheduler.dao.utils.PropertyUtils.getString(SPRING_DATASOURCE_DRIVER_CLASS_NAME); + String dataSourceDriverClass = PropertyUtils.getString(SPRING_DATASOURCE_DRIVER_CLASS_NAME); if (dataSourceDriverClass.equals(ORG_POSTGRESQL_DRIVER)) { properties.setProperty(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, conf.getString(ORG_QUARTZ_JOBSTORE_DRIVERDELEGATECLASS, PostgreSQLDelegate.class.getName())); } else { From 705715c3130be49a8bab9bc2d181b2ad26d453da Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 18 Mar 2021 09:37:17 +0800 Subject: [PATCH 04/13] [Improvement-5059][API] Verify that the new workflow definition version is the current version before removing it #5059 (#5060) * this process definition version is used, not allowed to delete. * update ProcessDefinitionVersionServiceTest test class. * update check has associated process definition sql. --- .../apache/dolphinscheduler/api/enums/Status.java | 4 +++- .../api/service/ProcessDefinitionService.java | 9 +++++++++ .../impl/ProcessDefinitionServiceImpl.java | 13 +++++++++++++ .../impl/ProcessDefinitionVersionServiceImpl.java | 15 +++++++++++++++ .../ProcessDefinitionVersionServiceTest.java | 13 +++++++++++++ .../dao/mapper/ProcessDefinitionMapper.java | 8 ++++++++ .../dao/mapper/ProcessDefinitionMapper.xml | 7 +++++++ .../definition/pages/list/_source/versions.vue | 4 ++-- 8 files changed, 70 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 1ec974445d..f77be9fa1a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -297,7 +297,9 @@ public enum Status { PLUGIN_INSTANCE_ALREADY_EXIT(110010, "plugin instance already exit", "该告警插件实例已存在"), LIST_PAGING_ALERT_PLUGIN_INSTANCE_ERROR(110011, "query plugin instance page error", "分页查询告警实例失败"), DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED(110012, "failed to delete the alert instance, there is an alarm group associated with this alert instance", - "删除告警实例失败，存在与此告警实例关联的警报组"); + "删除告警实例失败，存在与此告警实例关联的警报组"), + PROCESS_DEFINITION_VERSION_IS_USED(110013,"this process definition version is used","此工作流定义版本被使用"); + private final int code; private final String enMsg; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 82651dde74..be07225cd7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -274,5 +274,14 @@ public interface ProcessDefinitionService { */ Map switchProcessDefinitionVersion(User loginUser, String projectName , int processDefinitionId, long version); + + /** + * check has associated process definition + * + * @param processDefinitionId process definition id + * @param version version + * @return The query result has a specific process definition return true + */ + boolean checkHasAssociatedProcessDefinition(int processDefinitionId, long version); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 97cea0281a..4ab43fb37b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -1798,5 +1798,18 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro } } + /** + * check has associated process definition + * + * @param processDefinitionId process definition id + * @param version version + * @return The query result has a specific process definition return true + */ + @Override + public boolean checkHasAssociatedProcessDefinition(int processDefinitionId, long version) { + Integer hasAssociatedDefinitionId = processDefineMapper.queryHasAssociatedDefinitionByIdAndVersion(processDefinitionId, version); + return Objects.nonNull(hasAssociatedDefinitionId); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java index be7a3e93b0..988ab201fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.service.impl; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService; import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -56,6 +57,9 @@ public class ProcessDefinitionVersionServiceImpl extends BaseServiceImpl impleme @Autowired private ProjectMapper projectMapper; + @Autowired + private ProcessDefinitionService processDefinitionService; + /** * add the newest version of one process definition * @@ -178,8 +182,19 @@ public class ProcessDefinitionVersionServiceImpl extends BaseServiceImpl impleme if (resultStatus != Status.SUCCESS) { return checkResult; } + + // check has associated process definition + boolean hasAssociatedProcessDefinition = processDefinitionService.checkHasAssociatedProcessDefinition(processDefinitionId, version); + if (hasAssociatedProcessDefinition) { + putMsg(result, Status.PROCESS_DEFINITION_VERSION_IS_USED); + return result; + } + processDefinitionVersionMapper.deleteByProcessDefinitionIdAndVersion(processDefinitionId, version); putMsg(result, Status.SUCCESS); return result; } + + + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java index 1286877dad..f2a0e8deda 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java @@ -63,6 +63,9 @@ public class ProcessDefinitionVersionServiceTest { @Mock private ProjectServiceImpl projectService; + @Mock + private ProcessDefinitionService processDefinitionService; + @Test public void testAddProcessDefinitionVersion() { long expectedVersion = 5L; @@ -201,6 +204,8 @@ public class ProcessDefinitionVersionServiceTest { .thenReturn(1); Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)) .thenReturn(res); + Mockito.when(processDefinitionService.checkHasAssociatedProcessDefinition(processDefinitionId, version)) + .thenReturn(false); Map resultMap2 = processDefinitionVersionService.deleteByProcessDefinitionIdAndVersion( loginUser @@ -210,6 +215,14 @@ public class ProcessDefinitionVersionServiceTest { Assert.assertEquals(Status.SUCCESS, resultMap2.get(Constants.STATUS)); + Mockito.when(processDefinitionService.checkHasAssociatedProcessDefinition(processDefinitionId, version)) + .thenReturn(true); + Map resultMap3 = processDefinitionVersionService.deleteByProcessDefinitionIdAndVersion( + loginUser + , projectName + , processDefinitionId + , version); + Assert.assertEquals(Status.PROCESS_DEFINITION_VERSION_IS_USED, resultMap3.get(Constants.STATUS)); } /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java index 36c988776f..5f5f0f77dd 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java @@ -144,4 +144,12 @@ public interface ProcessDefinitionMapper extends BaseMapper { * @return project ids list */ List listProjectIds(); + + /** + * query has associated definition by id and version + * @param processDefinitionId process definition id + * @param version version + * @return definition id + */ + Integer queryHasAssociatedDefinitionByIdAndVersion(@Param("processDefinitionId") int processDefinitionId, @Param("version") long version); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index a24fd294a7..3abca4367f 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -139,4 +139,11 @@ set version = #{version} where id = #{processDefinitionId} + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue index b3046485a3..9677a1c991 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue @@ -50,7 +50,7 @@ :title="$t('Confirm Switch To This Version?')" @onConfirm="_mVersionSwitchProcessDefinitionVersion(scope.row)" > - + @@ -62,7 +62,7 @@ :title="$t('Delete?')" @onConfirm="_mVersionDeleteProcessDefinitionVersion(scope.row,scope.row.id)" > - + From 5856a12855328e67aeb6a2005f86b3c1081750a1 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Thu, 18 Mar 2021 10:03:58 +0800 Subject: [PATCH 05/13] [Improvement-4984][Worker] Refactor and Improve worker load balance (#4996) * [Improvement][Remote] Improve unit tests * [Improvement][Worker] Improve worker load balance * [Improvement][Worker] Fix code smells * [Improvement][Worker] Rename weight to hostWeight --- .../service/impl/WorkerGroupServiceImpl.java | 13 +- .../api/service/WorkerGroupServiceTest.java | 10 +- .../dolphinscheduler/common/Constants.java | 6 +- .../common/utils/ResInfo.java | 58 ++++--- .../dolphinscheduler/remote/utils/Host.java | 143 +++--------------- .../remote/NettyRemotingClientTest.java | 13 +- .../future}/ResponseFutureTest.java | 12 +- .../log}/RemoveTaskLogRequestCommandTest.java | 10 +- .../RemoveTaskLogResponseCommandTest.java | 9 +- .../remote}/utils/HostTest.java | 23 +-- .../remote/{ => utils}/NettyUtilTest.java | 5 +- .../dispatch/host/CommonHostManager.java | 48 +++--- .../dispatch/host/LowerWeightHostManager.java | 60 ++++---- .../dispatch/host/RandomHostManager.java | 11 +- .../dispatch/host/RoundRobinHostManager.java | 7 +- .../host/assign/AbstractSelector.java | 4 +- .../dispatch/host/assign/HostSelector.java | 6 +- .../dispatch/host/assign/HostWeight.java | 44 +++--- .../dispatch/host/assign/HostWorker.java | 77 ++++++++++ .../dispatch/host/assign/RandomSelector.java | 14 +- .../host/assign/RoundRobinSelector.java | 27 ++-- .../master/registry/MasterRegistry.java | 4 +- .../server/registry/HeartBeatTask.java | 30 +++- .../server/worker/config/WorkerConfig.java | 6 +- .../worker/registry/WorkerRegistry.java | 24 ++- .../server/zk/ZKMasterClient.java | 2 - .../dispatch/host/assign/HostWorkerTest.java | 43 ++++++ .../assign/LowerWeightRoundRobinTest.java | 22 ++- .../host/assign/RandomSelectorTest.java | 10 +- .../host/assign/RoundRobinSelectorTest.java | 36 ++--- .../master/registry/MasterRegistryTest.java | 3 +- .../service/zk/RegisterOperator.java | 4 +- pom.xml | 16 +- 33 files changed, 412 insertions(+), 388 deletions(-) rename dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/{ => command/future}/ResponseFutureTest.java (93%) rename dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/{ => command/log}/RemoveTaskLogRequestCommandTest.java (83%) rename dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/{ => command/log}/RemoveTaskLogResponseCommandTest.java (88%) rename {dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server => dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote}/utils/HostTest.java (56%) rename dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/{ => utils}/NettyUtilTest.java (92%) create mode 100644 dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index 7ce6a74d6b..e0282de221 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -163,18 +163,13 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro if (CollectionUtils.isEmpty(childrenNodes)) { continue; } - String timeStamp = childrenNodes.get(0); - for (int i = 0; i < childrenNodes.size(); i++) { - childrenNodes.set(i, Host.of(childrenNodes.get(i)).getAddressAndWeight()); - } - WorkerGroup wg = new WorkerGroup(); wg.setName(workerGroup); if (isPaging) { - wg.setIpList(childrenNodes); - String registeredIpValue = zookeeperCachedOperator.get(workerGroupPath + SLASH + timeStamp); - wg.setCreateTime(DateUtils.stringToDate(registeredIpValue.split(",")[6])); - wg.setUpdateTime(DateUtils.stringToDate(registeredIpValue.split(",")[7])); + wg.setIpList(childrenNodes.stream().map(node -> Host.of(node).getIp()).collect(Collectors.toList())); + String registeredValue = zookeeperCachedOperator.get(workerGroupPath + SLASH + childrenNodes.get(0)); + wg.setCreateTime(DateUtils.stringToDate(registeredValue.split(",")[6])); + wg.setUpdateTime(DateUtils.stringToDate(registeredValue.split(",")[7])); } workerGroups.add(wg); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java index bc95ad1e31..ce9f239d8a 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java @@ -70,13 +70,13 @@ public class WorkerGroupServiceTest { workerGroupStrList.add("test"); Mockito.when(zookeeperCachedOperator.getChildrenKeys(workerPath)).thenReturn(workerGroupStrList); - List defaultIpList = new ArrayList<>(); - defaultIpList.add("192.168.220.188:1234:100:1234567"); - defaultIpList.add("192.168.220.189:1234:100:1234567"); + List defaultAddressList = new ArrayList<>(); + defaultAddressList.add("192.168.220.188:1234"); + defaultAddressList.add("192.168.220.189:1234"); - Mockito.when(zookeeperCachedOperator.getChildrenKeys(workerPath + "/default")).thenReturn(defaultIpList); + Mockito.when(zookeeperCachedOperator.getChildrenKeys(workerPath + "/default")).thenReturn(defaultAddressList); - Mockito.when(zookeeperCachedOperator.get(workerPath + "/default" + "/" + defaultIpList.get(0))).thenReturn("0.01,0.17,0.03,25.83,8.0,1.0,2020-07-21 11:17:59,2020-07-21 14:39:20,0,13238"); + Mockito.when(zookeeperCachedOperator.get(workerPath + "/default" + "/" + defaultAddressList.get(0))).thenReturn("0.01,0.17,0.03,25.83,8.0,1.0,2020-07-21 11:17:59,2020-07-21 14:39:20,0,13238"); } /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 17412575c5..d6d571f145 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -385,6 +385,10 @@ public final class Constants { */ public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; + /** + * worker host weight + */ + public static final int DEFAULT_WORKER_HOST_WEIGHT = 100; /** * default log cache rows num,output when reach the number @@ -542,7 +546,7 @@ public final class Constants { * heartbeat for zk info length */ public static final int HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH = 10; - + public static final int HEARTBEAT_WITH_WEIGHT_FOR_ZOOKEEPER_INFO_LENGTH = 11; /** * jar diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java index 8a6ef1ea8c..8f533a022f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ResInfo.java @@ -14,7 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.model.Server; @@ -38,10 +40,8 @@ public class ResInfo { */ private double loadAverage; - public ResInfo(){} - - public ResInfo(double cpuUsage , double memoryUsage){ - this.cpuUsage = cpuUsage ; + public ResInfo(double cpuUsage, double memoryUsage) { + this.cpuUsage = cpuUsage; this.memoryUsage = memoryUsage; } @@ -81,35 +81,53 @@ public class ResInfo { * @param loadAverage load average * @return cpu and memory usage */ - public static String getResInfoJson(double cpuUsage , double memoryUsage,double loadAverage){ + public static String getResInfoJson(double cpuUsage, double memoryUsage, double loadAverage) { ResInfo resInfo = new ResInfo(cpuUsage,memoryUsage,loadAverage); return JSONUtils.toJsonString(resInfo); } - /** * parse heartbeat info for zk * @param heartBeatInfo heartbeat info * @return heartbeat info to Server */ - public static Server parseHeartbeatForZKInfo(String heartBeatInfo){ - if (StringUtils.isEmpty(heartBeatInfo)) { + public static Server parseHeartbeatForZKInfo(String heartBeatInfo) { + if (!isValidHeartbeatForZKInfo(heartBeatInfo)) { return null; } - String[] masterArray = heartBeatInfo.split(Constants.COMMA); - if(masterArray.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ - return null; + String[] parts = heartBeatInfo.split(Constants.COMMA); + Server server = new Server(); + server.setResInfo(getResInfoJson(Double.parseDouble(parts[0]), + Double.parseDouble(parts[1]), + Double.parseDouble(parts[2]))); + server.setCreateTime(DateUtils.stringToDate(parts[6])); + server.setLastHeartbeatTime(DateUtils.stringToDate(parts[7])); + //set process id + server.setId(Integer.parseInt(parts[9])); + return server; + } + /** + * is valid heartbeat info for zk + * @param heartBeatInfo heartbeat info + * @return heartbeat info is valid + */ + public static boolean isValidHeartbeatForZKInfo(String heartBeatInfo) { + if (StringUtils.isNotEmpty(heartBeatInfo)) { + String[] parts = heartBeatInfo.split(Constants.COMMA); + return parts.length == Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH + || parts.length == Constants.HEARTBEAT_WITH_WEIGHT_FOR_ZOOKEEPER_INFO_LENGTH; } - Server masterServer = new Server(); - masterServer.setResInfo(getResInfoJson(Double.parseDouble(masterArray[0]), - Double.parseDouble(masterArray[1]), - Double.parseDouble(masterArray[2]))); - masterServer.setCreateTime(DateUtils.stringToDate(masterArray[6])); - masterServer.setLastHeartbeatTime(DateUtils.stringToDate(masterArray[7])); - //set process id - masterServer.setId(Integer.parseInt(masterArray[9])); - return masterServer; + return false; + } + + /** + * is new heartbeat info for zk with weight + * @param parts heartbeat info parts + * @return heartbeat info is new with weight + */ + public static boolean isNewHeartbeatWithWeight(String[] parts) { + return parts.length == Constants.HEARTBEAT_WITH_WEIGHT_FOR_ZOOKEEPER_INFO_LENGTH; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java index 7e42984e49..359baefae6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java @@ -20,8 +20,6 @@ package org.apache.dolphinscheduler.remote.utils; import static org.apache.dolphinscheduler.common.Constants.COLON; import java.io.Serializable; -import java.util.Objects; -import java.util.StringJoiner; /** * server address @@ -43,21 +41,6 @@ public class Host implements Serializable { */ private int port; - /** - * weight - */ - private int weight; - - /** - * startTime - */ - private long startTime; - - /** - * workGroup - */ - private String workGroup; - public Host() { } @@ -67,21 +50,11 @@ public class Host implements Serializable { this.address = ip + COLON + port; } - public Host(String ip, int port, int weight, long startTime) { - this.ip = ip; - this.port = port; - this.address = ip + COLON + port; - this.weight = getWarmUpWeight(weight, startTime); - this.startTime = startTime; - } - - public Host(String ip, int port, int weight, long startTime, String workGroup) { - this.ip = ip; - this.port = port; - this.address = ip + COLON + port; - this.weight = getWarmUpWeight(weight, startTime); - this.workGroup = workGroup; - this.startTime = startTime; + public Host(String address) { + String[] parts = splitAddress(address); + this.ip = parts[0]; + this.port = Integer.parseInt(parts[1]); + this.address = address; } public String getAddress() { @@ -89,6 +62,9 @@ public class Host implements Serializable { } public void setAddress(String address) { + String[] parts = splitAddress(address); + this.ip = parts[0]; + this.port = Integer.parseInt(parts[1]); this.address = address; } @@ -101,22 +77,6 @@ public class Host implements Serializable { this.address = ip + COLON + port; } - public int getWeight() { - return weight; - } - - public void setWeight(int weight) { - this.weight = weight; - } - - public long getStartTime() { - return startTime; - } - - public void setStartTime(long startTime) { - this.startTime = startTime; - } - public int getPort() { return port; } @@ -126,12 +86,15 @@ public class Host implements Serializable { this.address = ip + COLON + port; } - public String getWorkGroup() { - return workGroup; - } - - public void setWorkGroup(String workGroup) { - this.workGroup = workGroup; + /** + * address convert host + * + * @param address address + * @return host + */ + public static Host of(String address) { + String[] parts = splitAddress(address); + return new Host(parts[0], Integer.parseInt(parts[1])); } /** @@ -140,37 +103,15 @@ public class Host implements Serializable { * @param address address * @return host */ - public static Host of(String address) { + public static String[] splitAddress(String address) { if (address == null) { throw new IllegalArgumentException("Host : address is null."); } String[] parts = address.split(COLON); - if (parts.length < 2) { + if (parts.length != 2) { throw new IllegalArgumentException(String.format("Host : %s illegal.", address)); } - Host host = null; - if (parts.length == 2) { - host = new Host(parts[0], Integer.parseInt(parts[1])); - } - if (parts.length == 4) { - host = new Host(parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2]), Long.parseLong(parts[3])); - } - return host; - } - - /** - * generate host string - * @param address address - * @param weight weight - * @param startTime startTime - * @return address:weight:startTime - */ - public static String generate(String address, int weight, long startTime) { - StringJoiner stringJoiner = new StringJoiner(COLON); - stringJoiner.add(address) - .add(String.valueOf(weight)) - .add(String.valueOf(startTime)); - return stringJoiner.toString(); + return parts; } /** @@ -181,54 +122,16 @@ public class Host implements Serializable { */ public static Boolean isOldVersion(String address) { String[] parts = address.split(COLON); - return parts.length != 2 && parts.length != 3; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Host host = (Host) o; - return Objects.equals(getAddress(), host.getAddress()); - } - - @Override - public int hashCode() { - return Objects.hash(getAddress()); + return parts.length != 2; } @Override public String toString() { return "Host{" + "address='" + address + '\'' - + ", weight=" + weight - + ", startTime=" + startTime - + ", workGroup='" + workGroup + '\'' + + ", ip='" + ip + '\'' + + ", port=" + port + '}'; } - /** - * warm up - */ - private int getWarmUpWeight(int weight, long startTime) { - long uptime = System.currentTimeMillis() - startTime; - //If the warm-up is not over, reduce the weight - if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { - return (int) (weight * ((float) uptime / Constants.WARM_UP_TIME)); - } - return weight; - } - - /** - * get address and weight - * - * @return address:weight - */ - public String getAddressAndWeight() { - return address + COLON + weight; - } } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java index cfc10b2acb..a3f6c7b582 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.remote; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.Ping; @@ -28,23 +27,25 @@ import org.apache.dolphinscheduler.remote.future.InvokeCallback; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Host; -import org.junit.Assert; -import org.junit.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; +import org.junit.Assert; +import org.junit.Test; + +import io.netty.channel.Channel; + /** * netty remote client test */ public class NettyRemotingClientTest { - /** - * test sned sync + * test send sync */ @Test - public void testSendSync(){ + public void testSendSync() { NettyServerConfig serverConfig = new NettyServerConfig(); NettyRemotingServer server = new NettyRemotingServer(serverConfig); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/ResponseFutureTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java similarity index 93% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/ResponseFutureTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java index 8836043257..0190c9946b 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/ResponseFutureTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java @@ -15,24 +15,24 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; - +package org.apache.dolphinscheduler.remote.command.future; import org.apache.dolphinscheduler.remote.future.InvokeCallback; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; -import org.junit.Assert; -import org.junit.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.junit.Assert; +import org.junit.Test; + public class ResponseFutureTest { @Test - public void testScanFutureTable(){ + public void testScanFutureTable() { ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service")); executorService.scheduleAtFixedRate(new Runnable() { @Override @@ -51,7 +51,7 @@ public class ResponseFutureTest { ResponseFuture future = new ResponseFuture(1, 2000, invokeCallback, null); try { latch.await(5000, TimeUnit.MILLISECONDS); - Assert.assertTrue(ResponseFuture.getFuture(1) == null); + Assert.assertNull(ResponseFuture.getFuture(1)); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java similarity index 83% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java index 37c21064c4..10646f9f02 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; +package org.apache.dolphinscheduler.remote.command.log; -import junit.framework.Assert; import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand; + import org.junit.Test; +import junit.framework.Assert; + public class RemoveTaskLogRequestCommandTest { @Test - public void testConvert2Command(){ + public void testConvert2Command() { RemoveTaskLogResponseCommand removeTaskLogResponseCommand = new RemoveTaskLogResponseCommand(); removeTaskLogResponseCommand.setStatus(true); Command command = removeTaskLogResponseCommand.convert2Command(122); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java similarity index 88% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java index aab0ad336f..87dd700b4b 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/RemoveTaskLogResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java @@ -15,17 +15,18 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; +package org.apache.dolphinscheduler.remote.command.log; -import junit.framework.Assert; import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand; + import org.junit.Test; +import junit.framework.Assert; + public class RemoveTaskLogResponseCommandTest { @Test - public void testConvert2Command(){ + public void testConvert2Command() { RemoveTaskLogRequestCommand removeTaskLogRequestCommand = new RemoveTaskLogRequestCommand(); removeTaskLogRequestCommand.setPath("/opt/zhangsan"); Command command = removeTaskLogRequestCommand.convert2Command(); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java similarity index 56% rename from dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java index 80ff11e0be..a9da007d5b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/HostTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/HostTest.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; - -import org.apache.dolphinscheduler.remote.utils.Host; +package org.apache.dolphinscheduler.remote.utils; import org.junit.Assert; import org.junit.Test; @@ -27,26 +25,13 @@ import org.junit.Test; */ public class HostTest { - @Test - public void testHostWarmUp() { - Host host = Host.of(("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000))); - Assert.assertEquals(50, host.getWeight()); - host = Host.of(("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 10 * 1000))); - Assert.assertEquals(100, host.getWeight()); - } - @Test public void testHost() { Host host = Host.of("192.158.2.2:22"); Assert.assertEquals(22, host.getPort()); + host.setAddress("127.0.0.1:8888"); + Assert.assertEquals("127.0.0.1", host.getIp()); + Assert.assertEquals(8888, host.getPort()); } - @Test - public void testGenerate() { - String address = "192.158.2.2:22"; - int weight = 100; - long startTime = System.currentTimeMillis(); - String generateHost = Host.generate(address, weight, startTime); - Assert.assertEquals(address + ":" + weight + ":" + startTime, generateHost); - } } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java similarity index 92% rename from dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java rename to dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java index e95dbddac9..a3e13850ac 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/utils/NettyUtilTest.java @@ -15,12 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote; +package org.apache.dolphinscheduler.remote.utils; import static org.apache.dolphinscheduler.remote.utils.Constants.OS_NAME; -import org.apache.dolphinscheduler.remote.utils.NettyUtils; - import org.junit.Assert; import org.junit.Test; @@ -31,7 +29,6 @@ import io.netty.channel.epoll.Epoll; */ public class NettyUtilTest { - @Test public void testUserEpoll() { if (OS_NAME.toLowerCase().contains("linux") && Epoll.isAvailable()) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java index 4a3d4bd9f1..7184b9e328 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java @@ -17,26 +17,32 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.ResInfo; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; +import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; /** * round robin host manager */ public abstract class CommonHostManager implements HostManager { - private final Logger logger = LoggerFactory.getLogger(CommonHostManager.class); + /** + * zookeeper registry center + */ + @Autowired + protected ZookeeperRegistryCenter registryCenter; /** * zookeeperNodeManager @@ -50,16 +56,15 @@ public abstract class CommonHostManager implements HostManager { * @return host */ @Override - public Host select(ExecutionContext context){ + public Host select(ExecutionContext context) { Host host = new Host(); Collection nodes = null; - /** - * executor type - */ + String workerGroup = context.getWorkerGroup(); + // executor type ExecutorType executorType = context.getExecutorType(); - switch (executorType){ + switch (executorType) { case WORKER: - nodes = zookeeperNodeManager.getWorkerGroupNodes(context.getWorkerGroup()); + nodes = zookeeperNodeManager.getWorkerGroupNodes(workerGroup); break; case CLIENT: break; @@ -67,21 +72,26 @@ public abstract class CommonHostManager implements HostManager { throw new IllegalArgumentException("invalid executorType : " + executorType); } - if(CollectionUtils.isEmpty(nodes)){ + if (nodes == null || nodes.isEmpty()) { return host; } - List candidateHosts = new ArrayList<>(nodes.size()); + List candidateHosts = new ArrayList<>(); nodes.forEach(node -> { - Host nodeHost=Host.of(node); - nodeHost.setWorkGroup(context.getWorkerGroup()); - candidateHosts.add(nodeHost); + String workerGroupPath = registryCenter.getWorkerGroupPath(workerGroup); + String heartbeat = registryCenter.getRegisterOperator().get(workerGroupPath + "/" + node); + int hostWeight = Constants.DEFAULT_WORKER_HOST_WEIGHT; + if (StringUtils.isNotEmpty(heartbeat)) { + String[] parts = heartbeat.split(Constants.COMMA); + if (ResInfo.isNewHeartbeatWithWeight(parts)) { + hostWeight = Integer.parseInt(parts[10]); + } + } + candidateHosts.add(HostWorker.of(node, hostWeight, workerGroup)); }); - - return select(candidateHosts); } - protected abstract Host select(Collection nodes); + protected abstract HostWorker select(Collection nodes); public void setZookeeperNodeManager(ZookeeperNodeManager zookeeperNodeManager) { this.zookeeperNodeManager = zookeeperNodeManager; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java index ac7d8b0ffc..ef249a08dc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java @@ -19,20 +19,20 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.ResInfo; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWeight; +import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.LowerWeightRoundRobin; -import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -40,8 +40,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import static org.apache.dolphinscheduler.common.Constants.COMMA; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * round robin host manager @@ -50,12 +53,6 @@ public class LowerWeightHostManager extends CommonHostManager { private final Logger logger = LoggerFactory.getLogger(LowerWeightHostManager.class); - /** - * zookeeper registry center - */ - @Autowired - private ZookeeperRegistryCenter registryCenter; - /** * round robin host manager */ @@ -82,7 +79,7 @@ public class LowerWeightHostManager extends CommonHostManager { private ScheduledExecutorService executorService; @PostConstruct - public void init(){ + public void init() { this.selector = new LowerWeightRoundRobin(); this.workerHostWeightsMap = new ConcurrentHashMap<>(); this.lock = new ReentrantLock(); @@ -103,20 +100,20 @@ public class LowerWeightHostManager extends CommonHostManager { * @return host */ @Override - public Host select(ExecutionContext context){ + public Host select(ExecutionContext context) { Set workerHostWeights = getWorkerHostWeights(context.getWorkerGroup()); - if(CollectionUtils.isNotEmpty(workerHostWeights)){ + if (CollectionUtils.isNotEmpty(workerHostWeights)) { return selector.select(workerHostWeights).getHost(); } return new Host(); } @Override - public Host select(Collection nodes) { + public HostWorker select(Collection nodes) { throw new UnsupportedOperationException("not support"); } - private void syncWorkerHostWeight(Map> workerHostWeights){ + private void syncWorkerHostWeight(Map> workerHostWeights) { lock.lock(); try { workerHostWeightsMap.clear(); @@ -126,7 +123,7 @@ public class LowerWeightHostManager extends CommonHostManager { } } - private Set getWorkerHostWeights(String workerGroup){ + private Set getWorkerHostWeights(String workerGroup) { lock.lock(); try { return workerHostWeightsMap.get(workerGroup); @@ -135,7 +132,7 @@ public class LowerWeightHostManager extends CommonHostManager { } } - class RefreshResourceTask implements Runnable{ + class RefreshResourceTask implements Runnable { @Override public void run() { @@ -143,35 +140,34 @@ public class LowerWeightHostManager extends CommonHostManager { Map> workerGroupNodes = zookeeperNodeManager.getWorkerGroupNodes(); Set>> entries = workerGroupNodes.entrySet(); Map> workerHostWeights = new HashMap<>(); - for(Map.Entry> entry : entries){ + for (Map.Entry> entry : entries) { String workerGroup = entry.getKey(); Set nodes = entry.getValue(); String workerGroupPath = registryCenter.getWorkerGroupPath(workerGroup); Set hostWeights = new HashSet<>(nodes.size()); - for(String node : nodes){ + for (String node : nodes) { String heartbeat = registryCenter.getRegisterOperator().get(workerGroupPath + "/" + node); - if(StringUtils.isNotEmpty(heartbeat) - && heartbeat.split(COMMA).length == Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ - String[] parts = heartbeat.split(COMMA); - + if (ResInfo.isValidHeartbeatForZKInfo(heartbeat)) { + String[] parts = heartbeat.split(Constants.COMMA); int status = Integer.parseInt(parts[8]); - if (status == Constants.ABNORMAL_NODE_STATUS){ + if (status == Constants.ABNORMAL_NODE_STATUS) { logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", Double.parseDouble(parts[3]) , Double.parseDouble(parts[2])); continue; } - double cpu = Double.parseDouble(parts[0]); double memory = Double.parseDouble(parts[1]); double loadAverage = Double.parseDouble(parts[2]); - HostWeight hostWeight = new HostWeight(Host.of(node), cpu, memory, loadAverage); + long startTime = DateUtils.stringToDate(parts[6]).getTime(); + int weight = ResInfo.isNewHeartbeatWithWeight(parts) ? Integer.parseInt(parts[10]) : Constants.DEFAULT_WORKER_HOST_WEIGHT; + HostWeight hostWeight = new HostWeight(HostWorker.of(node, weight, workerGroup), cpu, memory, loadAverage, startTime); hostWeights.add(hostWeight); } } workerHostWeights.put(workerGroup, hostWeights); } syncWorkerHostWeight(workerHostWeights); - } catch (Throwable ex){ + } catch (Throwable ex) { logger.error("RefreshResourceTask error", ex); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java index 241906a7b4..ad4f4aa6fd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java @@ -17,13 +17,11 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.RandomSelector; -import org.apache.dolphinscheduler.server.master.dispatch.host.assign.Selector; import java.util.Collection; - /** * round robin host manager */ @@ -32,17 +30,18 @@ public class RandomHostManager extends CommonHostManager { /** * selector */ - private final Selector selector; + private final RandomSelector selector; /** * set round robin */ - public RandomHostManager(){ + public RandomHostManager() { this.selector = new RandomSelector(); } @Override - public Host select(Collection nodes) { + public HostWorker select(Collection nodes) { return selector.select(nodes); } + } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java index ec1945e563..c6b173472f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java @@ -17,9 +17,8 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.RoundRobinSelector; -import org.apache.dolphinscheduler.server.master.dispatch.host.assign.Selector; import java.util.Collection; @@ -32,7 +31,7 @@ public class RoundRobinHostManager extends CommonHostManager { /** * selector */ - private final Selector selector; + private final RoundRobinSelector selector; /** * set round robin @@ -42,7 +41,7 @@ public class RoundRobinHostManager extends CommonHostManager { } @Override - public Host select(Collection nodes) { + public HostWorker select(Collection nodes) { return selector.select(nodes); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java index 8560da957e..087a5ff002 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java @@ -23,7 +23,7 @@ import java.util.Collection; /** * AbstractSelector */ -public abstract class AbstractSelector implements Selector{ +public abstract class AbstractSelector implements Selector { @Override public T select(Collection source) { @@ -40,6 +40,6 @@ public abstract class AbstractSelector implements Selector{ return doSelect(source); } - protected abstract T doSelect(Collection source); + protected abstract T doSelect(Collection source); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java index 145393e1f0..0be87e24cd 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostSelector.java @@ -28,9 +28,9 @@ public enum HostSelector { LOWERWEIGHT; - public static HostSelector of(String selector){ - for(HostSelector hs : values()){ - if(hs.name().equalsIgnoreCase(selector)){ + public static HostSelector of(String selector) { + for (HostSelector hs : values()) { + if (hs.name().equalsIgnoreCase(selector)) { return hs; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java index 839ebc85c7..9d7855f054 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java @@ -31,57 +31,55 @@ public class HostWeight { private final int LOAD_AVERAGE_FACTOR = 70; - private final Host host; + private final HostWorker hostWorker; private final double weight; private double currentWeight; - public HostWeight(Host host, double cpu, double memory, double loadAverage) { - this.weight = getWeight(cpu, memory, loadAverage, host); - this.host = host; - this.currentWeight = weight; - } - - public double getCurrentWeight() { - return currentWeight; + public HostWeight(HostWorker hostWorker, double cpu, double memory, double loadAverage, long startTime) { + this.hostWorker = hostWorker; + this.weight = calculateWeight(cpu, memory, loadAverage, startTime); + this.currentWeight = this.weight; } public double getWeight() { return weight; } + public double getCurrentWeight() { + return currentWeight; + } + public void setCurrentWeight(double currentWeight) { this.currentWeight = currentWeight; } + public HostWorker getHostWorker() { + return hostWorker; + } + public Host getHost() { - return host; + return (Host)hostWorker; } @Override public String toString() { return "HostWeight{" - + "host=" + host + + "hostWorker=" + hostWorker + ", weight=" + weight + ", currentWeight=" + currentWeight + '}'; } - private double getWeight(double cpu, double memory, double loadAverage, Host host) { - double calculateWeight = cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR; - return getWarmUpWeight(host, calculateWeight); - } - - /** - * If the warm-up is not over, add the weight - */ - private double getWarmUpWeight(Host host, double weight) { - long startTime = host.getStartTime(); + private double calculateWeight(double cpu, double memory, double loadAverage, long startTime) { + double calculatedWeight = cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR; long uptime = System.currentTimeMillis() - startTime; if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { - return weight * Constants.WARM_UP_TIME / uptime; + // If the warm-up is not over, add the weight + return calculatedWeight * Constants.WARM_UP_TIME / uptime; } - return weight; + return calculatedWeight; } + } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java new file mode 100644 index 0000000000..6515464b89 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorker.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.dispatch.host.assign; + +import org.apache.dolphinscheduler.remote.utils.Host; + +/** + * host worker + */ +public class HostWorker extends Host { + + /** + * host weight + */ + private int hostWeight; + + /** + * worker group + */ + private String workerGroup; + + public HostWorker(String ip, int port, int hostWeight, String workerGroup) { + super(ip, port); + this.hostWeight = hostWeight; + this.workerGroup = workerGroup; + } + + public HostWorker(String address, int hostWeight, String workerGroup) { + super(address); + this.hostWeight = hostWeight; + this.workerGroup = workerGroup; + } + + public int getHostWeight() { + return hostWeight; + } + + public void setHostWeight(int hostWeight) { + this.hostWeight = hostWeight; + } + + public String getWorkerGroup() { + return workerGroup; + } + + public void setWorkerGroup(String workerGroup) { + this.workerGroup = workerGroup; + } + + public static HostWorker of(String address, int hostWeight, String workerGroup) { + return new HostWorker(address, hostWeight, workerGroup); + } + + @Override + public String toString() { + return "Host{" + + "hostWeight=" + hostWeight + + ", workerGroup='" + workerGroup + '\'' + + '}'; + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java index 6975127b9a..2b7488a370 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Host; - import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -27,20 +25,20 @@ import java.util.concurrent.ThreadLocalRandom; /** * random selector */ -public class RandomSelector extends AbstractSelector { +public class RandomSelector extends AbstractSelector { @Override - public Host doSelect(final Collection source) { + public HostWorker doSelect(final Collection source) { - List hosts = new ArrayList<>(source); + List hosts = new ArrayList<>(source); int size = hosts.size(); int[] weights = new int[size]; int totalWeight = 0; int index = 0; - for (Host host : hosts) { - totalWeight += host.getWeight(); - weights[index] = host.getWeight(); + for (HostWorker host : hosts) { + totalWeight += host.getHostWeight(); + weights[index] = host.getHostWeight(); index++; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java index 34a79ac6e8..a5d93a684b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java @@ -16,20 +16,21 @@ */ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.springframework.stereotype.Service; - -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.springframework.stereotype.Service; + /** * Smooth Weight Round Robin */ @Service -public class RoundRobinSelector extends AbstractSelector { +public class RoundRobinSelector extends AbstractSelector { private ConcurrentMap> workGroupWeightMap = new ConcurrentHashMap<>(); @@ -69,12 +70,11 @@ public class RoundRobinSelector extends AbstractSelector { } - @Override - public Host doSelect(Collection source) { + public HostWorker doSelect(Collection source) { - List hosts = new ArrayList<>(source); - String key = hosts.get(0).getWorkGroup(); + List hosts = new ArrayList<>(source); + String key = hosts.get(0).getWorkerGroup(); ConcurrentMap map = workGroupWeightMap.get(key); if (map == null) { workGroupWeightMap.putIfAbsent(key, new ConcurrentHashMap<>()); @@ -84,13 +84,13 @@ public class RoundRobinSelector extends AbstractSelector { int totalWeight = 0; long maxCurrent = Long.MIN_VALUE; long now = System.currentTimeMillis(); - Host selectedHost = null; + HostWorker selectedHost = null; WeightedRoundRobin selectWeightRoundRobin = null; - for (Host host : hosts) { - String workGroupHost = host.getWorkGroup() + host.getAddress(); + for (HostWorker host : hosts) { + String workGroupHost = host.getWorkerGroup() + host.getAddress(); WeightedRoundRobin weightedRoundRobin = map.get(workGroupHost); - int weight = host.getWeight(); + int weight = host.getHostWeight(); if (weight < 0) { weight = 0; } @@ -117,7 +117,6 @@ public class RoundRobinSelector extends AbstractSelector { totalWeight += weight; } - if (!updateLock.get() && hosts.size() != map.size() && updateLock.compareAndSet(false, true)) { try { ConcurrentMap newMap = new ConcurrentHashMap<>(map); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java index e54fc84538..f77f00d53a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java @@ -98,8 +98,8 @@ public class MasterRegistry { }); int masterHeartbeatInterval = masterConfig.getMasterHeartbeatInterval(); HeartBeatTask heartBeatTask = new HeartBeatTask(startTime, - masterConfig.getMasterReservedMemory(), masterConfig.getMasterMaxCpuloadAvg(), + masterConfig.getMasterReservedMemory(), Sets.newHashSet(getMasterPath()), Constants.MASTER_PREFIX, zookeeperRegistryCenter); @@ -132,9 +132,7 @@ public class MasterRegistry { * get local address */ private String getLocalAddress() { - return NetUtils.getAddr(masterConfig.getListenPort()); - } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java index d5f7a6edd2..e4c3d6ab6d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java @@ -38,8 +38,9 @@ public class HeartBeatTask implements Runnable { private final Logger logger = LoggerFactory.getLogger(HeartBeatTask.class); private String startTime; - private double reservedMemory; private double maxCpuloadAvg; + private double reservedMemory; + private int hostWeight; // worker host weight private Set heartBeatPaths; private String serverType; private ZookeeperRegistryCenter zookeeperRegistryCenter; @@ -48,23 +49,38 @@ public class HeartBeatTask implements Runnable { protected IStoppable stoppable = null; public HeartBeatTask(String startTime, - double reservedMemory, double maxCpuloadAvg, + double reservedMemory, Set heartBeatPaths, String serverType, ZookeeperRegistryCenter zookeeperRegistryCenter) { this.startTime = startTime; - this.reservedMemory = reservedMemory; this.maxCpuloadAvg = maxCpuloadAvg; + this.reservedMemory = reservedMemory; this.heartBeatPaths = heartBeatPaths; + this.serverType = serverType; this.zookeeperRegistryCenter = zookeeperRegistryCenter; + } + + public HeartBeatTask(String startTime, + double maxCpuloadAvg, + double reservedMemory, + int hostWeight, + Set heartBeatPaths, + String serverType, + ZookeeperRegistryCenter zookeeperRegistryCenter) { + this.startTime = startTime; + this.maxCpuloadAvg = maxCpuloadAvg; + this.reservedMemory = reservedMemory; + this.hostWeight = hostWeight; + this.heartBeatPaths = heartBeatPaths; this.serverType = serverType; + this.zookeeperRegistryCenter = zookeeperRegistryCenter; } @Override public void run() { try { - // check dead or not in zookeeper for (String heartBeatPath : heartBeatPaths) { if (zookeeperRegistryCenter.checkIsDeadServer(heartBeatPath, serverType)) { @@ -94,8 +110,12 @@ public class HeartBeatTask implements Runnable { builder.append(startTime).append(Constants.COMMA); builder.append(DateUtils.dateToString(new Date())).append(Constants.COMMA); builder.append(status).append(COMMA); - //save process id + // save process id builder.append(OSUtils.getProcessID()); + // worker host weight + if (Constants.WORKER_PREFIX.equals(serverType)) { + builder.append(Constants.COMMA).append(hostWeight); + } for (String heartBeatPath : heartBeatPaths) { zookeeperRegistryCenter.getRegisterOperator().update(heartBeatPath, builder.toString()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 228f6ab755..0bd84f5e6d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -47,7 +47,7 @@ public class WorkerConfig { @Value("#{'${worker.groups:default}'.split(',')}") private Set workerGroups; - @Value("${worker.listen.port: 1234}") + @Value("${worker.listen.port:1234}") private int listenPort; @Value("${worker.host.weight:100}") @@ -119,8 +119,8 @@ public class WorkerConfig { return hostWeight; } - public void setHostWeight(int weight) { - this.hostWeight = weight; + public void setHostWeight(int hostWeight) { + this.hostWeight = hostWeight; } public String getAlertListenHost() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java index 06b72a5450..994fb58ff9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.server.registry.HeartBeatTask; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; @@ -34,6 +33,7 @@ import org.apache.curator.framework.state.ConnectionState; import java.util.Date; import java.util.Set; +import java.util.StringJoiner; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -120,12 +120,13 @@ public class WorkerRegistry { logger.info("worker node : {} registry to ZK {} successfully", address, workerZKPath); } - HeartBeatTask heartBeatTask = new HeartBeatTask(this.startTime, - this.workerConfig.getWorkerReservedMemory(), - this.workerConfig.getWorkerMaxCpuloadAvg(), + HeartBeatTask heartBeatTask = new HeartBeatTask(startTime, + workerConfig.getWorkerMaxCpuloadAvg(), + workerConfig.getWorkerReservedMemory(), + workerConfig.getHostWeight(), workerZkPaths, Constants.WORKER_PREFIX, - this.zookeeperRegistryCenter); + zookeeperRegistryCenter); this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, workerHeartbeatInterval, workerHeartbeatInterval, TimeUnit.SECONDS); logger.info("worker node : {} heartbeat interval {} s", address, workerHeartbeatInterval); @@ -150,22 +151,19 @@ public class WorkerRegistry { */ public Set getWorkerZkPaths() { Set workerZkPaths = Sets.newHashSet(); - String address = getLocalAddress(); String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath(); - int weight = workerConfig.getHostWeight(); - long workerStartTime = System.currentTimeMillis(); for (String workGroup : this.workerGroups) { - StringBuilder workerZkPathBuilder = new StringBuilder(100); - workerZkPathBuilder.append(workerZkPathPrefix).append(SLASH); + StringJoiner workerZkPathJoiner = new StringJoiner(SLASH); + workerZkPathJoiner.add(workerZkPathPrefix); if (StringUtils.isEmpty(workGroup)) { workGroup = DEFAULT_WORKER_GROUP; } // trim and lower case is need - workerZkPathBuilder.append(workGroup.trim().toLowerCase()).append(SLASH); - workerZkPathBuilder.append(Host.generate(address, weight, workerStartTime)); - workerZkPaths.add(workerZkPathBuilder.toString()); + workerZkPathJoiner.add(workGroup.trim().toLowerCase()); + workerZkPathJoiner.add(address); + workerZkPaths.add(workerZkPathJoiner.toString()); } return workerZkPaths; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index e813cdf1ae..0f30a5c155 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.master.MasterServer; @@ -308,7 +307,6 @@ public class ZKMasterClient extends AbstractZKClient { * @throws Exception exception */ private void failoverWorker(String workerHost, boolean needCheckWorkerAlive) throws Exception { - workerHost = Host.of(workerHost).getAddress(); logger.info("start worker[{}] failover ...", workerHost); List needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost); for (TaskInstance taskInstance : needFailoverTaskInstanceList) { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java new file mode 100644 index 0000000000..11f007ba21 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWorkerTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.dispatch.host.assign; + +import org.junit.Assert; +import org.junit.Test; + +public class HostWorkerTest { + + @Test + public void testHostWorker1() { + HostWorker hostWorker = new HostWorker("192.158.2.2", 11, 20, "default"); + Assert.assertEquals("192.158.2.2", hostWorker.getIp()); + Assert.assertEquals(11, hostWorker.getPort()); + Assert.assertEquals(20, hostWorker.getHostWeight()); + Assert.assertEquals("default", hostWorker.getWorkerGroup()); + } + + @Test + public void testHostWorker2() { + HostWorker hostWorker = HostWorker.of("192.158.2.2:22", 80, "default"); + Assert.assertEquals("192.158.2.2", hostWorker.getIp()); + Assert.assertEquals(22, hostWorker.getPort()); + Assert.assertEquals(80, hostWorker.getHostWeight()); + Assert.assertEquals("default", hostWorker.getWorkerGroup()); + } + +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java index fd5dda0873..f822f04d97 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java @@ -17,24 +17,20 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Host; - -import org.junit.Assert; -import org.junit.Test; - import java.util.ArrayList; import java.util.Collection; +import org.junit.Assert; +import org.junit.Test; public class LowerWeightRoundRobinTest { - @Test public void testSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(Host.of("192.158.2.1:11:100:" + (System.currentTimeMillis() - 60 * 8 * 1000)), 0.06, 0.44, 3.84)); - sources.add(new HostWeight(Host.of("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000)), 0.06, 0.56, 3.24)); - sources.add(new HostWeight(Host.of("192.158.2.3:33:100:" + (System.currentTimeMillis() - 60 * 2 * 1000)), 0.06, 0.80, 3.15)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, System.currentTimeMillis() - 60 * 2 * 1000)); LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); HostWeight result; @@ -55,10 +51,10 @@ public class LowerWeightRoundRobinTest { @Test public void testWarmUpSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(Host.of("192.158.2.1:11:100:" + (System.currentTimeMillis() - 60 * 8 * 1000)), 0.06, 0.44, 3.84)); - sources.add(new HostWeight(Host.of("192.158.2.2:22:100:" + (System.currentTimeMillis() - 60 * 5 * 1000)), 0.06, 0.44, 3.84)); - sources.add(new HostWeight(Host.of("192.158.2.3:33:100:" + (System.currentTimeMillis() - 60 * 3 * 1000)), 0.06, 0.44, 3.84)); - sources.add(new HostWeight(Host.of("192.158.2.4:33:100:" + (System.currentTimeMillis() - 60 * 11 * 1000)), 0.06, 0.44, 3.84)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.44, 3.84, System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.44, 3.84, System.currentTimeMillis() - 60 * 3 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.4:33", 100, "default"), 0.06, 0.44, 3.84, System.currentTimeMillis() - 60 * 11 * 1000)); LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); HostWeight result; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java index 14aa7b8f1f..2173ec88db 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java @@ -17,14 +17,11 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Host; +import java.util.Arrays; import org.junit.Assert; import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; - /** * random selector */ @@ -39,15 +36,14 @@ public class RandomSelectorTest { @Test public void testSelect1() { RandomSelector selector = new RandomSelector(); - Host result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 100, System.currentTimeMillis()), new Host("192.168.1.2", 80, 20, System.currentTimeMillis()))); + HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1:11", 100, "default"), new HostWorker("192.168.1.2:22", 80, "default"))); Assert.assertNotNull(result); } @Test public void testSelect() { RandomSelector selector = new RandomSelector(); - Host result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 100, System.currentTimeMillis()), new Host("192.168.1.1", 80, 20, System.currentTimeMillis()))); + HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1", 11, 100, "default"), new HostWorker("192.168.1.2:", 22, 20, "default"))); Assert.assertNotNull(result); - } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java index 1c9f4922a7..08badd5968 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java @@ -17,14 +17,12 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; -import org.apache.dolphinscheduler.remote.utils.Host; +import java.util.Arrays; +import java.util.List; import org.junit.Assert; import org.junit.Test; -import java.util.Arrays; -import java.util.List; - /** * round robin selector */ @@ -39,18 +37,16 @@ public class RoundRobinSelectorTest { @Test public void testSelect1() { RoundRobinSelector selector = new RoundRobinSelector(); - // dismiss of server warm-up time - long startTime = System.currentTimeMillis() - 60 * 10 * 1000; - List hostOneList = Arrays.asList( - new Host("192.168.1.1", 80, 20, startTime, "kris"), - new Host("192.168.1.2", 80, 10, startTime, "kris")); - - List hostTwoList = Arrays.asList( - new Host("192.168.1.1", 80, 20, startTime, "kris"), - new Host("192.168.1.2", 80, 10, startTime, "kris"), - new Host("192.168.1.3", 80, 10, startTime, "kris")); - - Host result; + List hostOneList = Arrays.asList( + new HostWorker("192.168.1.1", 80, 20, "kris"), + new HostWorker("192.168.1.2", 80, 10, "kris")); + + List hostTwoList = Arrays.asList( + new HostWorker("192.168.1.1", 80, 20, "kris"), + new HostWorker("192.168.1.2", 80, 10, "kris"), + new HostWorker("192.168.1.3", 80, 10, "kris")); + + HostWorker result; result = selector.select(hostOneList); Assert.assertEquals("192.168.1.1", result.getIp()); @@ -93,17 +89,15 @@ public class RoundRobinSelectorTest { result = selector.select(hostOneList); Assert.assertEquals("192.168.1.1", result.getIp()); - } @Test - public void testWarmUpRoundRobinSelector() { + public void testWeightRoundRobinSelector() { RoundRobinSelector selector = new RoundRobinSelector(); - Host result; + HostWorker result; result = selector.select( - Arrays.asList(new Host("192.168.1.1", 80, 20, System.currentTimeMillis() - 60 * 1000 * 2, "kris"), new Host("192.168.1.2", 80, 10, System.currentTimeMillis() - 60 * 1000 * 10, "kris"))); + Arrays.asList(new HostWorker("192.168.1.1", 11, 20, "kris"), new HostWorker("192.168.1.2", 22, 80, "kris"))); Assert.assertEquals("192.168.1.2", result.getIp()); - } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java index 9b62473930..8068ebd664 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.master.registry; import static org.apache.dolphinscheduler.common.Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; @@ -59,7 +60,7 @@ public class MasterRegistryTest { masterRegistry.registry(); String masterPath = zookeeperRegistryCenter.getMasterPath(); TimeUnit.SECONDS.sleep(masterConfig.getMasterHeartbeatInterval() + 2); //wait heartbeat info write into zk node - String masterNodePath = masterPath + "/" + (Constants.LOCAL_ADDRESS + ":" + masterConfig.getListenPort()); + String masterNodePath = masterPath + "/" + (NetUtils.getAddr(Constants.LOCAL_ADDRESS, masterConfig.getListenPort())); String heartbeat = zookeeperRegistryCenter.getRegisterOperator().get(masterNodePath); Assert.assertEquals(HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH, heartbeat.split(",").length); masterRegistry.unRegistry(); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/RegisterOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/RegisterOperator.java index 0fd4a4fa92..0c6ac6d0d4 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/RegisterOperator.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/zk/RegisterOperator.java @@ -69,10 +69,10 @@ public class RegisterOperator extends ZookeeperCachedOperator { } /** - * get host ip, string format: masterParentPath/ip + * get host ip:port, string format: parentPath/ip:port * * @param path path - * @return host ip, string format: masterParentPath/ip + * @return host ip:port, string format: parentPath/ip:port */ protected String getHostByEventDataPath(String path) { if (StringUtils.isEmpty(path)) { diff --git a/pom.xml b/pom.xml index 4662fc41ff..2d18af9c61 100644 --- a/pom.xml +++ b/pom.xml @@ -896,15 +896,15 @@ **/dao/datasource/MySQLDataSourceTest.java **/dao/entity/TaskInstanceTest.java **/dao/entity/UdfFuncTest.java - **/remote/JsonSerializerTest.java - **/remote/RemoveTaskLogResponseCommandTest.java - **/rpc/RpcTest.java - **/remote/RemoveTaskLogRequestCommandTest.java - **/remote/NettyRemotingClientTest.java - **/remote/NettyUtilTest.java - **/remote/ResponseFutureTest.java **/remote/command/alert/AlertSendRequestCommandTest.java **/remote/command/alert/AlertSendResponseCommandTest.java + **/remote/command/future/ResponseFutureTest.java + **/remote/command/log/RemoveTaskLogRequestCommandTest.java + **/remote/command/log/RemoveTaskLogResponseCommandTest.java + **/remote/utils/HostTest.java + **/remote/utils/NettyUtilTest.java + **/remote/NettyRemotingClientTest.java + **/rpc/RpcTest.java **/server/log/LoggerServerTest.java **/server/entity/SQLTaskExecutionContextTest.java **/server/log/MasterLogFilterTest.java @@ -919,6 +919,7 @@ **/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java **/server/master/dispatch/host/assign/RandomSelectorTest.java **/server/master/dispatch/host/assign/RoundRobinSelectorTest.java + **/server/master/dispatch/host/assign/HostWorkerTest.java **/server/master/register/MasterRegistryTest.java **/server/master/dispatch/host/assign/RoundRobinHostManagerTest.java **/server/master/AlertManagerTest.java @@ -935,7 +936,6 @@ **/server/register/ZookeeperRegistryCenterTest.java **/server/utils/DataxUtilsTest.java **/server/utils/ExecutionContextTestUtils.java - **/server/utils/HostTest.java **/server/utils/FlinkArgsUtilsTest.java **/server/utils/LogUtilsTest.java **/server/utils/MapReduceArgsUtilsTest.java From 29d42fd92d6720a8a0641e37923c6e6f38a5ae85 Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Thu, 18 Mar 2021 11:50:10 +0800 Subject: [PATCH 06/13] [Feature-#3805][server-master] Task parameter transfer (#5077) * fix out param format bug Co-authored-by: wangxj --- .../dolphinscheduler/common/Constants.java | 2 + .../master/runner/MasterExecThread.java | 13 +----- .../service/process/ProcessService.java | 46 +++++++++++++++---- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index d6d571f145..eb71239f48 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -469,6 +469,8 @@ public final class Constants { public static final String CMD_PARAM_START_PARAMS = "StartParams"; + public static final String CMD_PARAM_FATHER_PARAMS = "fatherParams"; + /** * complement data start date */ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 80bd3f7a9d..b7a4d00380 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -557,7 +557,7 @@ public class MasterExecThread implements Runnable { private void setProcessGlobal(TaskNode taskNode, TaskInstance taskInstance) { String globalParams = this.processInstance.getGlobalParams(); if (StringUtils.isNotEmpty(globalParams)) { - Map globalMap = getGlobalParamMap(globalParams); + Map globalMap = processService.getGlobalParamMap(globalParams); if (globalMap != null && globalMap.size() != 0) { setGlobalMapToTask(taskNode, taskInstance, globalMap); } @@ -586,17 +586,6 @@ public class MasterExecThread implements Runnable { } } - public Map getGlobalParamMap(String globalParams) { - List propList; - Map globalParamMap = new HashMap<>(); - if (StringUtils.isNotEmpty(globalParams)) { - propList = JSONUtils.toList(globalParams, Property.class); - globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); - } - - return globalParamMap; - } - private void submitPostNode(String parentNodeName) { Set submitTaskNodeList = DagHelper.parsePostNodes(parentNodeName, skipTaskNodeList, dag, completeTaskList); List taskInstances = new ArrayList<>(); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index a759c24e50..fe73eaedfb 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.service.process; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_FATHER_PARAMS; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS; import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_ID; @@ -586,14 +587,19 @@ public class ProcessService { private void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map cmdParam) { // get start params from command param - Map startParamMap = null; + Map startParamMap = new HashMap<>(); if (cmdParam != null && cmdParam.containsKey(Constants.CMD_PARAM_START_PARAMS)) { String startParamJson = cmdParam.get(Constants.CMD_PARAM_START_PARAMS); startParamMap = JSONUtils.toMap(startParamJson); } - + Map fatherParamMap = new HashMap<>(); + if (cmdParam != null && cmdParam.containsKey(Constants.CMD_PARAM_FATHER_PARAMS)) { + String fatherParamJson = cmdParam.get(Constants.CMD_PARAM_FATHER_PARAMS); + fatherParamMap = JSONUtils.toMap(fatherParamJson); + } + startParamMap.putAll(fatherParamMap); // set start param into global params - if (startParamMap != null && startParamMap.size() > 0 + if (startParamMap.size() > 0 && processDefinition.getGlobalParamMap() != null) { for (Map.Entry param : processDefinition.getGlobalParamMap().entrySet()) { String val = startParamMap.get(param.getKey()); @@ -1065,7 +1071,7 @@ public class ProcessService { /** * complement data needs transform parent parameter to child. */ - private String getSubWorkFlowParam(ProcessInstanceMap instanceMap, ProcessInstance parentProcessInstance) { + private String getSubWorkFlowParam(ProcessInstanceMap instanceMap, ProcessInstance parentProcessInstance,Map fatherParams) { // set sub work process command String processMapStr = JSONUtils.toJsonString(instanceMap); Map cmdParam = JSONUtils.toMap(processMapStr); @@ -1077,9 +1083,24 @@ public class ProcessService { cmdParam.put(CMDPARAM_COMPLEMENT_DATA_START_DATE, startTime); processMapStr = JSONUtils.toJsonString(cmdParam); } + if (fatherParams.size() != 0) { + cmdParam.put(CMD_PARAM_FATHER_PARAMS, JSONUtils.toJsonString(fatherParams)); + processMapStr = JSONUtils.toJsonString(cmdParam); + } return processMapStr; } + public Map getGlobalParamMap(String globalParams) { + List propList; + Map globalParamMap = new HashMap<>(); + if (StringUtils.isNotEmpty(globalParams)) { + propList = JSONUtils.toList(globalParams, Property.class); + globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); + } + + return globalParamMap; + } + /** * create sub work process command */ @@ -1089,9 +1110,18 @@ public class ProcessService { TaskInstance task) { CommandType commandType = getSubCommandType(parentProcessInstance, childInstance); TaskNode taskNode = JSONUtils.parseObject(task.getTaskJson(), TaskNode.class); - Map subProcessParam = JSONUtils.toMap(taskNode.getParams()); - Integer childDefineId = Integer.parseInt(subProcessParam.get(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_ID)); - String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance); + Map subProcessParam = JSONUtils.toMap(taskNode.getParams(), String.class, Object.class); + Integer childDefineId = Integer.parseInt(String.valueOf(subProcessParam.get(Constants.CMD_PARAM_SUB_PROCESS_DEFINE_ID))); + Object localParams = subProcessParam.get(Constants.LOCAL_PARAMS); + List allParam = JSONUtils.toList(JSONUtils.toJsonString(localParams), Property.class); + Map globalMap = this.getGlobalParamMap(parentProcessInstance.getGlobalParams()); + Map fatherParams = new HashMap<>(); + if (CollectionUtils.isNotEmpty(allParam)) { + for (Property info : allParam) { + fatherParams.put(info.getProp(), globalMap.get(info.getProp())); + } + } + String processParam = getSubWorkFlowParam(instanceMap, parentProcessInstance,fatherParams); return new Command( commandType, @@ -1601,7 +1631,7 @@ public class ProcessService { if (property == null) { continue; } - String value = row.get(paramName); + String value = String.valueOf(row.get(paramName)); if (StringUtils.isNotEmpty(value)) { property.setValue(value); info.setValue(value); From f94cfc620dfd0c51010a49134a073e3848c0bd7e Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Thu, 18 Mar 2021 18:34:42 +0800 Subject: [PATCH 07/13] [Feature][Worker] Add a configuration item to set whether the tenant is automatically created on Worker (#5007) --- .../dolphinscheduler/worker.properties.tpl | 3 ++ .../common/utils/OSUtils.java | 34 ++++++++++++++----- .../common/utils/OSUtilsTest.java | 16 ++++++--- .../server/worker/config/WorkerConfig.java | 12 +++---- .../processor/TaskExecuteProcessor.java | 4 +++ .../src/main/resources/worker.properties | 3 ++ 6 files changed, 53 insertions(+), 19 deletions(-) diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index ec0c4abb49..8f5907ede6 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -21,6 +21,9 @@ worker.exec.threads=${WORKER_EXEC_THREADS} # worker heartbeat interval worker.heartbeat.interval=${WORKER_HEARTBEAT_INTERVAL} +# worker tenant auto create +worker.tenant.auto.create=true + # only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 worker.max.cpuload.avg=${WORKER_MAX_CPULOAD_AVG} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 768c0f654a..92a06fe568 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -257,6 +257,23 @@ public class OSUtils { return users; } + /** + * create user + * + * @param userName user name + */ + public static void createUserIfAbsent(String userName) { + // if not exists this user, then create + taskLoggerThreadLocal.set(taskLoggerThreadLocal.get()); + if (!getUserList().contains(userName)) { + boolean isSuccess = createUser(userName); + String infoLog = String.format("create user %s %s", userName, isSuccess ? "success" : "fail"); + LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog); + LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog); + } + taskLoggerThreadLocal.remove(); + } + /** * create user * @@ -265,7 +282,7 @@ public class OSUtils { */ public static boolean createUser(String userName) { try { - String userGroup = OSUtils.getGroup(); + String userGroup = getGroup(); if (StringUtils.isEmpty(userGroup)) { String errorLog = String.format("%s group does not exist for this operating system.", userGroup); LoggerUtils.logError(Optional.ofNullable(logger), errorLog); @@ -304,7 +321,7 @@ public class OSUtils { String infoLog2 = String.format("execute cmd : %s", cmd); LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2); - OSUtils.exeCmd(cmd); + exeCmd(cmd); } /** @@ -315,7 +332,6 @@ public class OSUtils { * @throws IOException in case of an I/O error */ private static void createMacUser(String userName, String userGroup) throws IOException { - Optional optionalLogger = Optional.ofNullable(logger); Optional optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get()); @@ -327,13 +343,13 @@ public class OSUtils { String infoLog2 = String.format("create user command : %s", createUserCmd); LoggerUtils.logInfo(optionalLogger, infoLog2); LoggerUtils.logInfo(optionalTaskLogger, infoLog2); - OSUtils.exeCmd(createUserCmd); + exeCmd(createUserCmd); String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup); String infoLog3 = String.format("append user to group : %s", appendGroupCmd); LoggerUtils.logInfo(optionalLogger, infoLog3); LoggerUtils.logInfo(optionalTaskLogger, infoLog3); - OSUtils.exeCmd(appendGroupCmd); + exeCmd(appendGroupCmd); } /** @@ -352,13 +368,13 @@ public class OSUtils { String infoLog2 = String.format("execute create user command : %s", userCreateCmd); LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2); - OSUtils.exeCmd(userCreateCmd); + exeCmd(userCreateCmd); String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName); String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd); LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3); LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3); - OSUtils.exeCmd(appendGroupCmd); + exeCmd(appendGroupCmd); } /** @@ -472,9 +488,9 @@ public class OSUtils { */ public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory) { // system load average - double loadAverage = OSUtils.loadAverage(); + double loadAverage = loadAverage(); // system available physical memory - double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + double availablePhysicalMemorySize = availablePhysicalMemorySize(); if (loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory) { logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 83cacb758b..86da2684f7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -16,17 +16,19 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; + import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.dolphinscheduler.common.Constants; + +import java.io.IOException; +import java.util.List; + import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; - public class OSUtilsTest { private static final Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); @@ -75,6 +77,12 @@ public class OSUtilsTest { } } + @Test + public void createUserIfAbsent() { + OSUtils.createUserIfAbsent("test123"); + Assert.assertTrue("create user test123 success", true); + } + @Test public void testGetSudoCmd() { String cmd = "kill -9 1234"; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 0bd84f5e6d..1492899e84 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -35,8 +35,8 @@ public class WorkerConfig { @Value("${worker.heartbeat.interval:10}") private int workerHeartbeatInterval; - @Value("${worker.fetch.task.num:3}") - private int workerFetchTaskNum; + @Value("${worker.tenant.auto.create:false}") + private boolean workerTenantAutoCreate; @Value("${worker.max.cpuload.avg:-1}") private int workerMaxCpuloadAvg; @@ -88,12 +88,12 @@ public class WorkerConfig { this.workerHeartbeatInterval = workerHeartbeatInterval; } - public int getWorkerFetchTaskNum() { - return workerFetchTaskNum; + public boolean getWorkerTenantAutoCreate() { + return workerTenantAutoCreate; } - public void setWorkerFetchTaskNum(int workerFetchTaskNum) { - this.workerFetchTaskNum = workerFetchTaskNum; + public void setWorkerTenantAutoCreate(boolean workerTenantAutoCreate) { + this.workerTenantAutoCreate = workerTenantAutoCreate; } public double getWorkerReservedMemory() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index aafd7a1524..cfd2c3fe2f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.Preconditions; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -146,6 +147,9 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { FileUtils.taskLoggerThreadLocal.set(taskLogger); try { FileUtils.createWorkDirIfAbsent(execLocalPath); + if (workerConfig.getWorkerTenantAutoCreate()) { + OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode()); + } } catch (Throwable ex) { String errorLog = String.format("create execLocalPath : %s", execLocalPath); LoggerUtils.logError(Optional.of(logger), errorLog, ex); diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index 7fd11fe6c2..88c15a6d5d 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -21,6 +21,9 @@ # worker heartbeat interval #worker.heartbeat.interval=10 +# worker tenant auto create +#worker.tenant.auto.create=false + # only less than cpu avg load, worker server can work. default value -1: the number of cpu cores * 2 #worker.max.cpuload.avg=-1 From 086e71644163a0bb44e4bd1c5f7f16abc383ab1a Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Thu, 18 Mar 2021 22:49:12 +0800 Subject: [PATCH 08/13] [Improvement][Dao] SQL optimization - check exist #5069 (#5070) --- .../service/impl/AlertGroupServiceImpl.java | 5 ++- .../impl/AlertPluginInstanceServiceImpl.java | 5 +-- .../api/service/impl/QueueServiceImpl.java | 8 ++--- .../service/impl/ResourcesServiceImpl.java | 5 +-- .../api/service/impl/TenantServiceImpl.java | 7 ++-- .../impl/WorkFlowLineageServiceImpl.java | 3 +- .../api/service/AlertGroupServiceTest.java | 2 +- .../AlertPluginInstanceServiceTest.java | 2 +- .../api/service/QueueServiceTest.java | 31 ++++++++--------- .../api/service/ResourcesServiceTest.java | 6 ++-- .../api/service/TenantServiceTest.java | 6 ++-- .../common/utils/BooleanUtils.java | 33 +++++++++++++++++++ .../dolphinscheduler/dao/PluginDao.java | 3 +- .../dao/mapper/AlertGroupMapper.java | 7 ++++ .../dao/mapper/AlertPluginInstanceMapper.java | 7 ++++ .../dao/mapper/QueueMapper.java | 7 ++++ .../dao/mapper/ResourceMapper.java | 11 +++++++ .../dao/mapper/TenantMapper.java | 7 ++++ .../dao/mapper/UserMapper.java | 9 ++++- .../dao/mapper/AlertGroupMapper.xml | 7 ++++ .../dao/mapper/AlertPluginInstanceMapper.xml | 6 ++++ .../dao/mapper/QueueMapper.xml | 12 ++++++- .../dao/mapper/ResourceMapper.xml | 8 +++++ .../dao/mapper/TenantMapper.xml | 5 +++ .../dao/mapper/UserMapper.xml | 7 +++- .../dao/mapper/AlertGroupMapperTest.java | 9 +++++ .../mapper/AlertPluginInstanceMapperTest.java | 8 +++++ .../dao/mapper/QueueMapperTest.java | 25 ++++++++++---- .../dao/mapper/ResourceMapperTest.java | 10 ++++++ .../dao/mapper/TenantMapperTest.java | 12 ++++++- .../dao/mapper/UserMapperTest.java | 10 ++++++ 31 files changed, 234 insertions(+), 49 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java index 90df9a09a9..3831ad0892 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java @@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.AlertGroupService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.BooleanUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.User; @@ -207,7 +207,6 @@ public class AlertGroupServiceImpl extends BaseServiceImpl implements AlertGroup */ @Override public boolean existGroupName(String groupName) { - List alertGroup = alertGroupMapper.queryByGroupName(groupName); - return CollectionUtils.isNotEmpty(alertGroup); + return BooleanUtils.isTrue(alertGroupMapper.existGroupName(groupName)); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java index 95171edf06..ea4664e9a5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.BooleanUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; @@ -82,7 +83,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A Map result = new HashMap<>(); - if (CollectionUtils.isNotEmpty(alertPluginInstanceMapper.queryByInstanceName(alertPluginInstance.getInstanceName()))) { + if (BooleanUtils.isTrue(alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()))) { putMsg(result, Status.PLUGIN_INSTANCE_ALREADY_EXIT); return result; } @@ -183,7 +184,7 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A @Override public boolean checkExistPluginInstanceName(String pluginInstanceName) { - return CollectionUtils.isNotEmpty(alertPluginInstanceMapper.queryByInstanceName(pluginInstanceName)); + return BooleanUtils.isTrue(alertPluginInstanceMapper.existInstanceName(pluginInstanceName)); } @Override diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java index 002794dc56..8042a364e9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java @@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.BooleanUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.User; @@ -270,7 +270,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { * @return true if the queue not exists, otherwise return false */ private boolean checkQueueExist(String queue) { - return CollectionUtils.isNotEmpty(queueMapper.queryAllQueueList(queue, null)); + return BooleanUtils.isTrue(queueMapper.existQueue(queue, null)); } /** @@ -281,7 +281,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { * @return true if the queue name not exists, otherwise return false */ private boolean checkQueueNameExist(String queueName) { - return CollectionUtils.isNotEmpty(queueMapper.queryAllQueueList(null, queueName)); + return BooleanUtils.isTrue(queueMapper.existQueue(null, queueName)); } /** @@ -293,7 +293,7 @@ public class QueueServiceImpl extends BaseServiceImpl implements QueueService { * @return true if need to update user */ private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) { - return !oldQueue.equals(newQueue) && CollectionUtils.isNotEmpty(userMapper.queryUserListByQueue(oldQueue)); + return !oldQueue.equals(newQueue) && BooleanUtils.isTrue(userMapper.existUser(oldQueue)); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index f61996223e..7337b332f9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; +import org.apache.dolphinscheduler.common.utils.BooleanUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; @@ -251,8 +252,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * @return true if resource exists */ private boolean checkResourceExists(String fullName, int userId, int type) { - List resources = resourcesMapper.queryResourceList(fullName, userId, type); - return resources != null && !resources.isEmpty(); + Boolean existResource = resourcesMapper.existResource(fullName, userId, type); + return BooleanUtils.isTrue(existResource); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 9deef7d035..8a3a945d10 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.BooleanUtils; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -95,7 +96,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService } if (checkTenantExists(tenantCode)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode); + putMsg(result, Status.OS_TENANT_CODE_EXIST, tenantCode); return result; } @@ -344,7 +345,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService * @return ture if the tenant code exists, otherwise return false */ private boolean checkTenantExists(String tenantCode) { - List tenants = tenantMapper.queryByTenantCode(tenantCode); - return CollectionUtils.isNotEmpty(tenants); + Boolean existTenant = tenantMapper.existTenant(tenantCode); + return BooleanUtils.isTrue(existTenant); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java index 22c7622ab8..8cf16cdc7f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.service.impl; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation; import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper; @@ -56,7 +57,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF for (int id : ids) { sourceIds.addAll(ids); List workFlowRelationsTmp = workFlowLineageMapper.querySourceTarget(id); - if (workFlowRelationsTmp != null && !workFlowRelationsTmp.isEmpty()) { + if (CollectionUtils.isNotEmpty(workFlowRelationsTmp)) { Set idsTmp = new HashSet<>(); for (WorkFlowRelation workFlowRelation:workFlowRelationsTmp) { if (!sourceIds.contains(workFlowRelation.getTargetWorkFlowId())) { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java index 85115bbc28..0f5843bca0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java @@ -156,7 +156,7 @@ public class AlertGroupServiceTest { //group name not exist boolean result = alertGroupService.existGroupName(groupName); Assert.assertFalse(result); - Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList()); + Mockito.when(alertGroupMapper.existGroupName(groupName)).thenReturn(true); //group name exist result = alertGroupService.existGroupName(groupName); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java index ae3896218c..071d59bfe8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceServiceTest.java @@ -157,7 +157,7 @@ public class AlertPluginInstanceServiceTest { @Test public void testCreate() { - Mockito.when(alertPluginInstanceMapper.queryByInstanceName("test")).thenReturn(alertPluginInstances); + Mockito.when(alertPluginInstanceMapper.existInstanceName("test")).thenReturn(true); Map result = alertPluginInstanceService.create(user, 1, "test", uiParams); Assert.assertEquals(Status.PLUGIN_INSTANCE_ALREADY_EXIT, result.get(Constants.STATUS)); Mockito.when(alertPluginInstanceMapper.insert(Mockito.any())).thenReturn(1); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java index 10c6d486e7..6d48da87b9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java @@ -76,7 +76,7 @@ public class QueueServiceTest { } @Test - public void testQueryList(){ + public void testQueryList() { Mockito.when(queueMapper.selectList(null)).thenReturn(getQueueList()); Map result = queueService.queryList(getLoginUser()); @@ -86,7 +86,7 @@ public class QueueServiceTest { } @Test - public void testQueryListPage(){ + public void testQueryListPage() { IPage page = new Page<>(1,10); page.setTotal(1L); @@ -98,7 +98,7 @@ public class QueueServiceTest { Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); } @Test - public void testCreateQueue(){ + public void testCreateQueue() { // queue is null Map result = queueService.createQueue(getLoginUser(),null,queueName); @@ -114,13 +114,13 @@ public class QueueServiceTest { Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); } + @Test - public void testUpdateQueue(){ + public void testUpdateQueue() { Mockito.when(queueMapper.selectById(1)).thenReturn(getQueue()); - Mockito.when(queueMapper.queryAllQueueList("test", null)).thenReturn(getQueueList()); - Mockito.when(queueMapper.queryAllQueueList(null, "test")).thenReturn(getQueueList()); - Mockito.when(userMapper.queryUserListByQueue(queueName)).thenReturn(getUserList()); + Mockito.when(queueMapper.existQueue("test", null)).thenReturn(true); + Mockito.when(queueMapper.existQueue(null, "test")).thenReturn(true); // not exist Map result = queueService.updateQueue(getLoginUser(),0,"queue",queueName); @@ -144,11 +144,12 @@ public class QueueServiceTest { Assert.assertEquals(Status.SUCCESS.getCode(),((Status)result.get(Constants.STATUS)).getCode()); } + @Test - public void testVerifyQueue(){ + public void testVerifyQueue() { - Mockito.when(queueMapper.queryAllQueueList(queueName, null)).thenReturn(getQueueList()); - Mockito.when(queueMapper.queryAllQueueList(null, queueName)).thenReturn(getQueueList()); + Mockito.when(queueMapper.existQueue(queueName, null)).thenReturn(true); + Mockito.when(queueMapper.existQueue(null, queueName)).thenReturn(true); //queue null Result result = queueService.verifyQueue(null,queueName); @@ -175,13 +176,13 @@ public class QueueServiceTest { logger.info(result.toString()); Assert.assertEquals(result.getCode().intValue(), Status.SUCCESS.getCode()); - } + /** * create admin user * @return */ - private User getLoginUser(){ + private User getLoginUser() { User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); @@ -189,7 +190,7 @@ public class QueueServiceTest { return loginUser; } - private List getUserList(){ + private List getUserList() { List list = new ArrayList<>(); list.add(getLoginUser()); return list; @@ -200,7 +201,7 @@ public class QueueServiceTest { * get queue * @return */ - private Queue getQueue(){ + private Queue getQueue() { Queue queue = new Queue(); queue.setId(1); queue.setQueue(queueName); @@ -208,7 +209,7 @@ public class QueueServiceTest { return queue; } - private List getQueueList(){ + private List getQueueList() { List queueList = new ArrayList<>(); queueList.add(getQueue()); return queueList; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 8478b1b2bc..89bd3df243 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -169,7 +169,7 @@ public class ResourcesServiceTest { Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(resourcesMapper.queryResourceList("/directoryTest", 0, 0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.existResource("/directoryTest", 0, 0)).thenReturn(true); result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/"); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); @@ -227,7 +227,7 @@ public class ResourcesServiceTest { Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); //RESOURCE_EXIST - Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(true); result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); @@ -344,7 +344,7 @@ public class ResourcesServiceTest { User user = new User(); user.setId(1); - Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest.jar", 0, 0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest.jar", 0, 0)).thenReturn(true); Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index bdd38df421..ee85d260cc 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -79,7 +79,7 @@ public class TenantServiceTest { public void testCreateTenant() { User loginUser = getLoginUser(); - Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); + Mockito.when(tenantMapper.existTenant(tenantCode)).thenReturn(true); try { //check tenantCode Map result = @@ -90,7 +90,7 @@ public class TenantServiceTest { //check exist result = tenantService.createTenant(loginUser, tenantCode, 1, "TenantServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + Assert.assertEquals(Status.OS_TENANT_CODE_EXIST, result.get(Constants.STATUS)); // success result = tenantService.createTenant(loginUser, "test", 1, "TenantServiceTest"); @@ -186,7 +186,7 @@ public class TenantServiceTest { @Test public void testVerifyTenantCode() { - Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); + Mockito.when(tenantMapper.existTenant(tenantCode)).thenReturn(true); // tenantCode not exist Result result = tenantService.verifyTenantCode("s00000000000l887888885554444sfjdskfjslakslkdf"); logger.info(result.toString()); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java new file mode 100644 index 0000000000..5873556494 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/BooleanUtils.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +public class BooleanUtils { + + public static boolean isTrue(Boolean bool) { + if (bool == null) { + return false; + } else { + return bool; + } + } + + public static boolean isNotTrue(Boolean bool) { + return !isTrue(bool); + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java index 93df36aadc..646307496b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao; import static java.util.Objects.requireNonNull; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper; @@ -64,7 +65,7 @@ public class PluginDao extends AbstractBaseDao { requireNonNull(pluginDefine.getPluginType(), "pluginType is null"); List pluginDefineList = pluginDefineMapper.queryByNameAndType(pluginDefine.getPluginName(), pluginDefine.getPluginType()); - if (pluginDefineList == null || pluginDefineList.size() == 0) { + if (CollectionUtils.isEmpty(pluginDefineList)) { return pluginDefineMapper.insert(pluginDefine); } PluginDefine currPluginDefine = pluginDefineList.get(0); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java index ebfa5b42ec..8026b9a89f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java @@ -48,6 +48,13 @@ public interface AlertGroupMapper extends BaseMapper { */ List queryByGroupName(@Param("groupName") String groupName); + /** + * Judge whether the alert group exist + * @param groupName groupName + * @return if exist return true else return null + */ + Boolean existGroupName(@Param("groupName") String groupName); + /** * query by userId * @param userId userId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapper.java index 618a8774c7..12da99c3a0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapper.java @@ -44,4 +44,11 @@ public interface AlertPluginInstanceMapper extends BaseMapper queryByInstanceName(@Param("instanceName")String instanceName); + /** + * + * @param instanceName instanceName + * @return if exist return true else return null + */ + Boolean existInstanceName(@Param("instanceName") String instanceName); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java index 3be65b86d9..027bfd2b52 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java @@ -46,4 +46,11 @@ public interface QueueMapper extends BaseMapper { List queryAllQueueList(@Param("queue") String queue, @Param("queueName") String queueName); + /** + * check the target queue exist + * @param queue queue + * @param queueName queueName + * @return true if exist else return null + */ + Boolean existQueue(@Param("queue") String queue, @Param("queueName") String queueName); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index f58cc7d496..96b203963b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -144,4 +144,15 @@ public interface ResourceMapper extends BaseMapper { * @return update num */ int batchUpdateResource(@Param("resourceList") List resourceList); + + /** + * check resource exist + * @param fullName full name + * @param userId userId + * @param type type + * @return true if exist else return null + */ + Boolean existResource(@Param("fullName") String fullName, + @Param("userId") int userId, + @Param("type") int type); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java index e201805e79..45c824ce89 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java @@ -50,4 +50,11 @@ public interface TenantMapper extends BaseMapper { */ IPage queryTenantPaging(IPage page, @Param("searchVal") String searchVal); + + /** + * check tenant exist + * @param tenantCode tenantCode + * @return true if exist else return null + */ + Boolean existTenant(@Param("tenantCode") String tenantCode); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java index 1468f10b90..3e766389da 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java @@ -100,7 +100,14 @@ public interface UserMapper extends BaseMapper { * @param queueName queue name * @return user list */ - List queryUserListByQueue(@Param("queueName") String queueName); + List queryUserListByQueue(@Param("queue") String queueName); + + /** + * check the user exist + * @param queueName queue name + * @return true if exist else return null + */ + Boolean existUser(@Param("queue") String queue); /** * update user with old queue diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.xml index 90fd8a72a7..8a7d3a57e8 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.xml @@ -38,6 +38,13 @@ from t_ds_alertgroup where group_name=#{groupName} + + + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/QueueMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/QueueMapper.xml index 564dd0354f..0d75c6e364 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/QueueMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/QueueMapper.xml @@ -43,5 +43,15 @@ and queue_name =#{queueName} - + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml index 880e3260a5..c67fd7291c 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml @@ -166,4 +166,12 @@ #{i} + + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml index db3a282846..5c1575e706 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TenantMapper.xml @@ -54,4 +54,9 @@ order by t.update_time desc + diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml index f4263eb54b..32768a8639 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml @@ -103,7 +103,12 @@ select from t_ds_user - where queue = #{queueName} + where queue = #{queue} + + update t_ds_user diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java index db42a2f57b..9db5d2b9f2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -161,6 +162,14 @@ public class AlertGroupMapperTest { compareAlertGroups(alertGroupMap, alertGroupList); } + @Test + public void testExistGroupName() { + String groupName = "testGroup"; + createAlertGroups(1, groupName); + + Assert.assertTrue(alertGroupMapper.existGroupName(groupName)); + } + /** * test query all group list */ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapperTest.java index 6ebcde39f9..1688e0ec41 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertPluginInstanceMapperTest.java @@ -67,6 +67,14 @@ public class AlertPluginInstanceMapperTest { AlertGroup alertGroup = testAlertGroupList.get(0); } + @Test + public void testExistInstanceName() { + String instanceName = "test_instance"; + Assert.assertNull(alertPluginInstanceMapper.existInstanceName(instanceName)); + createAlertPluginInstance(); + Assert.assertTrue(alertPluginInstanceMapper.existInstanceName(instanceName)); + } + /** * insert * diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index a1e1fdaf7a..8d44f8878e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -14,12 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.mapper; +package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.utils.BooleanUtils; import org.apache.dolphinscheduler.dao.entity.Queue; + +import java.util.Date; +import java.util.List; + import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -29,8 +35,6 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; @RunWith(SpringRunner.class) @SpringBootTest @@ -47,7 +51,7 @@ public class QueueMapperTest { * insert * @return Queue */ - private Queue insertOne(){ + private Queue insertOne() { //insertOne Queue queue = new Queue(); queue.setQueueName("queue"); @@ -62,7 +66,7 @@ public class QueueMapperTest { * test update */ @Test - public void testUpdate(){ + public void testUpdate() { //insertOne Queue queue = insertOne(); queue.setCreateTime(new Date()); @@ -75,7 +79,7 @@ public class QueueMapperTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { Queue queue = insertOne(); int delete = queueMapper.deleteById(queue.getId()); Assert.assertEquals(1, delete); @@ -123,4 +127,13 @@ public class QueueMapperTest { queues = queueMapper.queryAllQueueList(null, queue.getQueueName()); Assert.assertNotEquals(queues.size(), 0); } + + @Test + public void existQueue() { + Assert.assertNull(queueMapper.existQueue("queue", null)); + Assert.assertNull(queueMapper.existQueue(null, "queue")); + Queue queue = insertOne(); + Assert.assertTrue(BooleanUtils.isTrue(queueMapper.existQueue(queue.getQueue(), null))); + Assert.assertTrue(BooleanUtils.isTrue(queueMapper.existQueue(null, queue.getQueueName()))); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index d36a26ffd3..f06e4fcd9b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -412,4 +412,14 @@ public class ResourceMapperTest { Assert.fail("batch update resource data error"); } } + + @Test + public void existResourceTest() { + String fullName = "/ut-resource"; + int userId = 111; + int type = ResourceType.FILE.getCode(); + Assert.assertNull(resourceMapper.existResource(fullName, userId, type)); + insertOne(); + Assert.assertTrue(resourceMapper.existResource(fullName, userId, type)); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java index a1860709bc..b8a97e15c6 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java @@ -54,6 +54,7 @@ public class TenantMapperTest { Tenant tenant = new Tenant(); tenant.setCreateTime(new Date()); tenant.setUpdateTime(new Date()); + tenant.setTenantCode("test_code"); tenantMapper.insert(tenant); return tenant; } @@ -122,6 +123,8 @@ public class TenantMapperTest { Tenant tenant = insertOne(); tenant.setTenantCode("ut code"); tenantMapper.updateById(tenant); + List tenantList = tenantMapper.queryByTenantCode("ut code"); + Assert.assertEquals(1, tenantList.size()); } /** @@ -139,11 +142,18 @@ public class TenantMapperTest { tenant.setTenantCode("ut code"); tenant.setQueueId(queue.getId()); tenantMapper.updateById(tenant); - Page page = new Page(1,3); + Page page = new Page(1, 3); //tenant.getTenantCode() used instead of tenant.getTenantName() IPage tenantIPage = tenantMapper.queryTenantPaging(page, tenant.getTenantCode()); Assert.assertNotEquals(tenantIPage.getTotal(), 0); } + + public void testExistTenant() { + String tenantCode = "test_code"; + Assert.assertNull(tenantMapper.existTenant(tenantCode)); + insertOne(); + Assert.assertTrue(tenantMapper.existTenant(tenantCode)); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java index 7f7ceb480a..e573ad77ee 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java @@ -74,6 +74,8 @@ public class UserMapperTest { user.setCreateTime(new Date()); user.setTenantId(1); user.setUpdateTime(new Date()); + user.setQueueName("test_queue"); + user.setQueue("queue"); userMapper.insert(user); return user; } @@ -312,4 +314,12 @@ public class UserMapperTest { Assert.assertEquals(userToken, user); } + + @Test + public void testExistUser() { + String queueName = "queue"; + Assert.assertNull(userMapper.existUser(queueName)); + insertOne(); + Assert.assertTrue(userMapper.existUser(queueName)); + } } From 9008fa4b0c43a615270f080e9d8c3c2a324bee9c Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Fri, 19 Mar 2021 11:57:46 +0800 Subject: [PATCH 09/13] [Fix-5089][UI] Fix code mirror cannot be displayed normally in sqoop task (#5090) * [Improvement][UI] Rename from-mirror to form-mirror * [Fix-5089][UI] Fix code mirror cannot be displayed normally in sqoop task --- .../js/conf/home/pages/dag/_source/dag.scss | 2 +- .../formModel/tasks/_source/jsonBox.vue | 4 +-- .../formModel/tasks/_source/scriptBox.vue | 4 +-- .../dag/_source/formModel/tasks/datax.vue | 4 +-- .../dag/_source/formModel/tasks/python.vue | 2 +- .../dag/_source/formModel/tasks/shell.vue | 2 +- .../pages/dag/_source/formModel/tasks/sql.vue | 2 +- .../dag/_source/formModel/tasks/sqoop.vue | 28 +++++++++++++++++-- .../components/conditions/conditions.vue | 2 +- .../src/sass/common/index.scss | 2 +- 10 files changed, 37 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss index 41a44bc17a..08f2feccbe 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.scss @@ -546,7 +546,7 @@ svg path:hover { } } -.from-mirror { +.form-mirror { width: 100%; position: relative; z-index: 0; diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/jsonBox.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/jsonBox.vue index 04375e616a..9b7f53e0cf 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/jsonBox.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/_source/jsonBox.vue @@ -18,7 +18,7 @@
-
+
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index d66c9e6819..4c61ed266e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -19,7 +19,7 @@
{{$t('Script')}}
-
+
@@ -223,10 +223,10 @@