From cb01b0deb1c5966e3c7d3395f6acd70a784f6b92 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Thu, 29 Jul 2021 18:31:57 +0800 Subject: [PATCH] [1.3.7-prepare][Improvement] Impove text description, OSUtilsTest and reduce heap memory for create/upgrade-dolphinscheduler.sh (#5890) * Rename NORAML_NODE_STATUS to NORMAL_NODE_STATUS * Fix OSUtilsTest * Optimize comments in properties * Reduce heap size to 64m in create-dolphinscheduler.sh and upgrade-dolphinscheduler.sh * [Improvement][License] Update oshi version in release-docs/LICENSE * Add test prefix for test method --- .../1.3.6/configuration/dolphin-master.xml | 2 +- .../1.3.6/configuration/dolphin-worker.xml | 2 +- .../dolphinscheduler/master.properties.tpl | 2 +- .../dolphinscheduler/worker.properties.tpl | 2 +- .../dolphinscheduler/common/Constants.java | 2 +- .../common/utils/OSUtils.java | 6 +-- .../src/main/resources/common.properties | 2 +- .../common/os/OSUtilsTest.java | 49 +++++++------------ .../common/utils/OSUtilsTest.java | 4 +- dolphinscheduler-dist/release-docs/LICENSE | 2 +- .../server/registry/HeartBeatTask.java | 2 +- .../src/main/resources/master.properties | 2 +- .../src/main/resources/worker.properties | 2 +- script/create-dolphinscheduler.sh | 2 +- script/upgrade-dolphinscheduler.sh | 2 +- 15 files changed, 36 insertions(+), 47 deletions(-) diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml index e7b2df2ea0..989fdea74d 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-master.xml @@ -30,7 +30,7 @@ int - master execute thread number to limit process instances + master execute thread number to limit process instances in parallel diff --git a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml index cd16f83bdb..f10f5c20b7 100644 --- a/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml +++ b/ambari_plugin/common-services/DOLPHIN/1.3.6/configuration/dolphin-worker.xml @@ -30,7 +30,7 @@ int - worker execute thread number to limit task instances + worker execute thread number to limit task instances in parallel diff --git a/docker/build/conf/dolphinscheduler/master.properties.tpl b/docker/build/conf/dolphinscheduler/master.properties.tpl index 3c79c0174c..046d5c15ff 100644 --- a/docker/build/conf/dolphinscheduler/master.properties.tpl +++ b/docker/build/conf/dolphinscheduler/master.properties.tpl @@ -18,7 +18,7 @@ # master listen port #master.listen.port=5678 -# master execute thread number to limit process instances +# master execute thread number to limit process instances in parallel master.exec.threads=${MASTER_EXEC_THREADS} # master execute task number in parallel per process instance diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index 7d266d7698..eb28ca8fb5 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -18,7 +18,7 @@ # worker listener port #worker.listen.port=1234 -# worker execute thread number to limit task instances +# worker execute thread number to limit task instances in parallel worker.exec.threads=${WORKER_EXEC_THREADS} # worker heartbeat interval, the unit is second diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index e0a96d4ac7..864af98bc9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -995,7 +995,7 @@ public final class Constants { */ public static final String PLUGIN_JAR_SUFFIX = ".jar"; - public static final int NORAML_NODE_STATUS = 0; + public static final int NORMAL_NODE_STATUS = 0; public static final int ABNORMAL_NODE_STATUS = 1; public static final String START_TIME = "start time"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index e5612c3863..a3868ac8e2 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -88,7 +88,7 @@ public class OSUtils { */ public static double availablePhysicalMemorySize() { GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024; + double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024; DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); @@ -101,9 +101,9 @@ public class OSUtils { * Keep 2 decimal * @return available Physical Memory Size, unit: G */ - public static double totalMemorySize() { + public static double totalPhysicalMemorySize() { GlobalMemory memory = hal.getMemory(); - double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; + double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 5abb907dd1..31498474cb 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -39,7 +39,7 @@ login.user.keytab.path=/opt/hdfs.headless.keytab # kerberos expire time, the unit is hour kerberos.expire.time=2 -# resource view suffixss +# resource view suffixs #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index bca3972350..e86fbaac77 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -19,15 +19,11 @@ package org.apache.dolphinscheduler.common.os; import org.apache.dolphinscheduler.common.utils.OSUtils; -import java.math.RoundingMode; -import java.text.DecimalFormat; - +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.hardware.GlobalMemory; - /** * OSUtilsTest */ @@ -36,40 +32,33 @@ public class OSUtilsTest { private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); @Test - public void memoryUsage() { - logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239 + public void testMemoryUsage() { + double memoryUsage = OSUtils.memoryUsage(); + logger.info("memoryUsage : {}", memoryUsage); + Assert.assertTrue(memoryUsage >= 0.0); } @Test - public void availablePhysicalMemorySize() { - logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize()); - logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10); + public void testPhysicalMemorySize() { + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize(); + logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize); + logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize); + Assert.assertTrue(availablePhysicalMemorySize >= 0.0); + Assert.assertTrue(totalPhysicalMemorySize >= 0.0); } - @Test - public void loadAverage() { - logger.info("memoryUsage : {}", OSUtils.loadAverage()); - } - - - private void printMemory(GlobalMemory memory) { - logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() ); + public void testLoadAverage() { + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage : {}", loadAverage); + Assert.assertTrue(loadAverage >= 0.0); } - @Test - public void cpuUsage() throws Exception { - logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - Thread.sleep(1000L); - logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - + public void testCpuUsage() { double cpuUsage = OSUtils.cpuUsage(); - - DecimalFormat df = new DecimalFormat("0.00"); - - df.setRoundingMode(RoundingMode.HALF_UP); - - logger.info("cpuUsage1 : {}", df.format(cpuUsage)); + logger.info("cpuUsage : {}", cpuUsage); + Assert.assertTrue(cpuUsage >= 0.0); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 28d589ec33..61382060fe 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -38,8 +38,8 @@ public class OSUtilsTest { public void testOSMetric(){ double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); Assert.assertTrue(availablePhysicalMemorySize > 0.0f); - double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize > 0.0f); + double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize(); + Assert.assertTrue(totalPhysicalMemorySize > 0.0f); double loadAverage = OSUtils.loadAverage(); logger.info("loadAverage {}", loadAverage); double memoryUsage = OSUtils.memoryUsage(); diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 8157349494..49091b3c57 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -443,7 +443,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. core-3.1.1 https://mvnrepository.com/artifact/org.eclipse.jdt/core/3.1.1 Eclipse Public License v1.0 logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1 logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1 - oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0 + oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0 junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0 h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0 diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java index c7e1edd30f..ccdd44538e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java @@ -75,7 +75,7 @@ public class HeartBeatTask implements Runnable { double loadAverage = OSUtils.loadAverage(); double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - int status = Constants.NORAML_NODE_STATUS; + int status = Constants.NORMAL_NODE_STATUS; if (loadAverage > maxCpuloadAvg || availablePhysicalMemorySize < reservedMemory) { logger.warn("current cpu load average {} is too high or available memory {}G is too low, under max.cpuload.avg={} and reserved.memory={}G", loadAverage, availablePhysicalMemorySize, maxCpuloadAvg, reservedMemory); diff --git a/dolphinscheduler-server/src/main/resources/master.properties b/dolphinscheduler-server/src/main/resources/master.properties index 3363d1f120..cc45622132 100644 --- a/dolphinscheduler-server/src/main/resources/master.properties +++ b/dolphinscheduler-server/src/main/resources/master.properties @@ -18,7 +18,7 @@ # master listen port #master.listen.port=5678 -# master execute thread number to limit process instances +# master execute thread number to limit process instances in parallel #master.exec.threads=100 # master execute task number in parallel per process instance diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index c19f439f43..bb822a5714 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -18,7 +18,7 @@ # worker listener port #worker.listen.port=1234 -# worker execute thread number to limit task instances +# worker execute thread number to limit task instances in parallel #worker.exec.threads=100 # worker heartbeat interval, the unit is second diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh index 7ab8cbcf17..9ca9499431 100755 --- a/script/create-dolphinscheduler.sh +++ b/script/create-dolphinscheduler.sh @@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh index 6944e3a132..42ba839d56 100755 --- a/script/upgrade-dolphinscheduler.sh +++ b/script/upgrade-dolphinscheduler.sh @@ -26,7 +26,7 @@ export JAVA_HOME=$JAVA_HOME export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/* -export DOLPHINSCHEDULER_OPTS="-server -Xmx1g -Xms1g -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" +export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" export STOP_TIMEOUT=5 CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler