From e92e29ef9a126d1d5660011545056d4dd806d105 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Mon, 19 Apr 2021 10:24:54 +0800 Subject: [PATCH] [Fix-5309][Common] Fix the wrong memory (#5312) * [Fix-5309][Common] Fix the wrong memory * [Fix-5309][Common] Fix unit test and remove unused code --- .../dolphinscheduler/common/Constants.java | 10 ---- .../common/utils/OSUtils.java | 35 ++------------ .../common/os/OSUtilsTest.java | 46 ++++++++----------- .../common/utils/OSUtilsTest.java | 40 +++------------- dolphinscheduler-dist/release-docs/LICENSE | 2 +- pom.xml | 2 +- tools/dependencies/known-dependencies.txt | 3 +- 7 files changed, 32 insertions(+), 106 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 0a46a35580..0bf071deb0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -380,21 +380,11 @@ public final class Constants { */ public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2; - /** - * master reserved memory - */ - public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; - /** * worker cpu load */ public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2; - /** - * worker reserved memory - */ - public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; - /** * worker host weight */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 09b4e06d9f..af62033a74 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -17,11 +17,8 @@ package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.shell.ShellExecutor; -import org.apache.commons.configuration.Configuration; - import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; @@ -85,7 +82,7 @@ public class OSUtils { */ public static double memoryUsage() { GlobalMemory memory = hal.getMemory(); - double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10; + double memoryUsage = (memory.getTotal() - memory.getAvailable()) * 1.0 / memory.getTotal(); DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); @@ -101,12 +98,11 @@ public class OSUtils { */ public static double availablePhysicalMemorySize() { GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024; + double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024; DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); return Double.parseDouble(df.format(availablePhysicalMemorySize)); - } /** @@ -116,13 +112,13 @@ public class OSUtils { * * @return available Physical Memory Size, unit: G */ - public static double totalMemorySize() { + public static double totalPhysicalMemorySize() { GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; + double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(availablePhysicalMemorySize)); + return Double.parseDouble(df.format(totalPhysicalMemorySize)); } /** @@ -504,25 +500,4 @@ public class OSUtils { } } - /** - * check memory and cpu usage - * - * @param conf conf - * @param isMaster is master - * @return check memory and cpu usage - */ - public static Boolean checkResource(Configuration conf, Boolean isMaster) { - double systemCpuLoad; - double systemReservedMemory; - - if (Boolean.TRUE.equals(isMaster)) { - systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); - systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); - } else { - systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); - systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); - } - return checkResource(systemCpuLoad, systemReservedMemory); - } - } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index b2fd887d3d..d6f3ffd024 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.os; import org.apache.dolphinscheduler.common.utils.OSUtils; + +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.hardware.GlobalMemory; - -import java.math.RoundingMode; -import java.text.DecimalFormat; /** @@ -34,42 +32,34 @@ public class OSUtilsTest { private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); - @Test public void memoryUsage() { - logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239 + double memoryUsage = OSUtils.memoryUsage(); + logger.info("memoryUsage : {}", memoryUsage); + Assert.assertTrue(memoryUsage >= 0.0); } @Test - public void availablePhysicalMemorySize() { - logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize()); - logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10); + public void physicalMemorySize() { + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize(); + logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize); + logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize); + Assert.assertTrue(availablePhysicalMemorySize >= 0.0); + Assert.assertTrue(totalPhysicalMemorySize >= 0.0); } - @Test public void loadAverage() { - logger.info("memoryUsage : {}", OSUtils.loadAverage()); - } - - - private void printMemory(GlobalMemory memory) { - logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() ); + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage : {}", loadAverage); + Assert.assertTrue(loadAverage >= 0.0); } - @Test - public void cpuUsage() throws Exception { - logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - Thread.sleep(1000L); - logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - + public void cpuUsage() { double cpuUsage = OSUtils.cpuUsage(); - - DecimalFormat df = new DecimalFormat("0.00"); - - df.setRoundingMode(RoundingMode.HALF_UP); - - logger.info("cpuUsage1 : {}", df.format(cpuUsage)); + logger.info("cpuUsage : {}", cpuUsage); + Assert.assertTrue(cpuUsage >= 0.0); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 86da2684f7..8467cec9b4 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -16,11 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.Constants; - -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.PropertiesConfiguration; - import java.io.IOException; import java.util.List; @@ -40,12 +35,12 @@ public class OSUtilsTest { } @Test - public void testOSMetric(){ + public void testOSMetric() { if (!OSUtils.isWindows()) { double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); Assert.assertTrue(availablePhysicalMemorySize >= 0.0d); - double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize >= 0.0d); + double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize(); + Assert.assertTrue(totalPhysicalMemorySize >= 0.0d); double loadAverage = OSUtils.loadAverage(); logger.info("loadAverage {}", loadAverage); double memoryUsage = OSUtils.memoryUsage(); @@ -92,7 +87,7 @@ public class OSUtilsTest { @Test public void exeCmd() { - if(OSUtils.isMacOS() || !OSUtils.isWindows()){ + if (OSUtils.isMacOS() || !OSUtils.isWindows()) { try { String result = OSUtils.exeCmd("echo helloWorld"); Assert.assertEquals("helloWorld\n",result); @@ -102,39 +97,16 @@ public class OSUtilsTest { } } @Test - public void getProcessID(){ + public void getProcessID() { int processId = OSUtils.getProcessID(); Assert.assertNotEquals(0, processId); } @Test - public void checkResource(){ + public void checkResource() { boolean resource = OSUtils.checkResource(100,0); Assert.assertTrue(resource); resource = OSUtils.checkResource(0,Double.MAX_VALUE); Assert.assertFalse(resource); - - Configuration configuration = new PropertiesConfiguration(); - - configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,100); - configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,0); - resource = OSUtils.checkResource(configuration,true); - Assert.assertTrue(resource); - - configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,0); - configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,Double.MAX_VALUE); - resource = OSUtils.checkResource(configuration,true); - Assert.assertFalse(resource); - - configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,100); - configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,0); - resource = OSUtils.checkResource(configuration,false); - Assert.assertTrue(resource); - - configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,0); - configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,Double.MAX_VALUE); - resource = OSUtils.checkResource(configuration,false); - Assert.assertFalse(resource); - } } diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index f9c7e98084..eb7399740f 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -471,7 +471,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. aspectjweaver 1.9.2:https://mvnrepository.com/artifact/org.aspectj/aspectjweaver/1.9.2, EPL 1.0 logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1 logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1 - oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0 + oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0 junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0 h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0 diff --git a/pom.xml b/pom.xml index 71d679975d..06835a1a35 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ 42.1.4 2.1.0 2.4 - 3.5.0 + 3.9.1 0.1.52 6.1.0.jre8 0.238.1 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index f7598317d5..d403b16910 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -179,7 +179,7 @@ mybatis-spring-2.0.2.jar netty-3.6.2.Final.jar netty-all-4.1.52.Final.jar opencsv-2.3.jar -oshi-core-3.5.0.jar +oshi-core-3.9.1.jar paranamer-2.3.jar parquet-hadoop-bundle-1.8.1.jar plexus-cipher-1.7.jar @@ -237,7 +237,6 @@ swagger-annotations-1.5.20.jar swagger-bootstrap-ui-1.9.3.jar swagger-models-1.5.24.jar tephra-api-0.6.0.jar -threetenbp-1.3.6.jar transaction-api-1.1.jar validation-api-2.0.1.Final.jar wagon-provider-api-2.2.jar