[Fix-5309][Common] Fix the wrong memory (#5312)

* [Fix-5309][Common] Fix the wrong memory

* [Fix-5309][Common] Fix unit test and remove unused code

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -380,21 +380,11 @@ public final class Constants {
      */
     public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
 
-    /**
-     * master reserved memory
-     */
-    public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
-
     /**
      * worker cpu load
      */
     public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
 
-    /**
-     * worker reserved memory
-     */
-    public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
-
     /**
      * worker host weight
      */

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java

@@ -17,11 +17,8 @@
 
 package org.apache.dolphinscheduler.common.utils;
 
-import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.shell.ShellExecutor;
 
-import org.apache.commons.configuration.Configuration;
-
 import java.io.BufferedReader;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -85,7 +82,7 @@ public class OSUtils {
      */
     public static double memoryUsage() {
         GlobalMemory memory = hal.getMemory();
-        double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10;
+        double memoryUsage = (memory.getTotal() - memory.getAvailable()) * 1.0 / memory.getTotal();
 
         DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
         df.setRoundingMode(RoundingMode.HALF_UP);
@@ -101,12 +98,11 @@ public class OSUtils {
      */
     public static double availablePhysicalMemorySize() {
         GlobalMemory memory = hal.getMemory();
-        double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024;
+        double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
 
         DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
         df.setRoundingMode(RoundingMode.HALF_UP);
         return Double.parseDouble(df.format(availablePhysicalMemorySize));
-
     }
 
     /**
@@ -116,13 +112,13 @@ public class OSUtils {
      *
      * @return available Physical Memory Size, unit: G
      */
-    public static double totalMemorySize() {
+    public static double totalPhysicalMemorySize() {
         GlobalMemory memory = hal.getMemory();
-        double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
+        double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
 
         DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
         df.setRoundingMode(RoundingMode.HALF_UP);
-        return Double.parseDouble(df.format(availablePhysicalMemorySize));
+        return Double.parseDouble(df.format(totalPhysicalMemorySize));
     }
 
     /**
@@ -504,25 +500,4 @@ public class OSUtils {
         }
     }
 
-    /**
-     * check memory and cpu usage
-     *
-     * @param conf conf
-     * @param isMaster is master
-     * @return check memory and cpu usage
-     */
-    public static Boolean checkResource(Configuration conf, Boolean isMaster) {
-        double systemCpuLoad;
-        double systemReservedMemory;
-
-        if (Boolean.TRUE.equals(isMaster)) {
-            systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD);
-            systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY);
-        } else {
-            systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD);
-            systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY);
-        }
-        return checkResource(systemCpuLoad, systemReservedMemory);
-    }
-
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java

@@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.os;
 
 
 import org.apache.dolphinscheduler.common.utils.OSUtils;
+
+import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import oshi.hardware.GlobalMemory;
-
-import java.math.RoundingMode;
-import java.text.DecimalFormat;
 
 
 /**
@@ -34,42 +32,34 @@ public class OSUtilsTest {
 
     private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class);
 
-
     @Test
     public void memoryUsage() {
-        logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239
+        double memoryUsage = OSUtils.memoryUsage();
+        logger.info("memoryUsage : {}", memoryUsage);
+        Assert.assertTrue(memoryUsage >= 0.0);
     }
 
     @Test
-    public void availablePhysicalMemorySize() {
+    public void physicalMemorySize() {
-        logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize());
+        double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
-        logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10);
+        double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
+        logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize);
+        logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize);
+        Assert.assertTrue(availablePhysicalMemorySize >= 0.0);
+        Assert.assertTrue(totalPhysicalMemorySize >= 0.0);
     }
 
-
     @Test
     public void loadAverage() {
-        logger.info("memoryUsage : {}", OSUtils.loadAverage());
+        double loadAverage = OSUtils.loadAverage();
-    }
+        logger.info("loadAverage : {}", loadAverage);
-
+        Assert.assertTrue(loadAverage >= 0.0);
-
-    private void printMemory(GlobalMemory memory) {
-        logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() );
     }
 
-
     @Test
-    public void cpuUsage() throws Exception {
+    public void cpuUsage() {
-        logger.info("cpuUsage : {}", OSUtils.cpuUsage());
-        Thread.sleep(1000L);
-        logger.info("cpuUsage : {}", OSUtils.cpuUsage());
-
         double cpuUsage = OSUtils.cpuUsage();
-
+        logger.info("cpuUsage : {}", cpuUsage);
-        DecimalFormat df = new DecimalFormat("0.00");
+        Assert.assertTrue(cpuUsage >= 0.0);
-
-        df.setRoundingMode(RoundingMode.HALF_UP);
-
-        logger.info("cpuUsage1 : {}", df.format(cpuUsage));
     }
 }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java

@@ -16,11 +16,6 @@
  */
 package org.apache.dolphinscheduler.common.utils;
 
-import org.apache.dolphinscheduler.common.Constants;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.PropertiesConfiguration;
-
 import java.io.IOException;
 import java.util.List;
 
@@ -40,12 +35,12 @@ public class OSUtilsTest {
     }
 
     @Test
-    public void testOSMetric(){
+    public void testOSMetric() {
         if (!OSUtils.isWindows()) {
             double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
             Assert.assertTrue(availablePhysicalMemorySize >= 0.0d);
-            double totalMemorySize = OSUtils.totalMemorySize();
+            double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
-            Assert.assertTrue(totalMemorySize >= 0.0d);
+            Assert.assertTrue(totalPhysicalMemorySize >= 0.0d);
             double loadAverage = OSUtils.loadAverage();
             logger.info("loadAverage {}", loadAverage);
             double memoryUsage = OSUtils.memoryUsage();
@@ -92,7 +87,7 @@ public class OSUtilsTest {
 
     @Test
     public void exeCmd() {
-        if(OSUtils.isMacOS() || !OSUtils.isWindows()){
+        if (OSUtils.isMacOS() || !OSUtils.isWindows()) {
             try {
                 String result = OSUtils.exeCmd("echo helloWorld");
                 Assert.assertEquals("helloWorld\n",result);
@@ -102,39 +97,16 @@ public class OSUtilsTest {
         }
     }
     @Test
-    public void getProcessID(){
+    public void getProcessID() {
         int processId = OSUtils.getProcessID();
         Assert.assertNotEquals(0, processId);
     }
     @Test
-    public void checkResource(){
+    public void checkResource() {
         boolean resource = OSUtils.checkResource(100,0);
         Assert.assertTrue(resource);
         resource = OSUtils.checkResource(0,Double.MAX_VALUE);
         Assert.assertFalse(resource);
-
-        Configuration configuration = new PropertiesConfiguration();
-
-        configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,100);
-        configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,0);
-        resource = OSUtils.checkResource(configuration,true);
-        Assert.assertTrue(resource);
-
-        configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,0);
-        configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,Double.MAX_VALUE);
-        resource = OSUtils.checkResource(configuration,true);
-        Assert.assertFalse(resource);
-
-        configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,100);
-        configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,0);
-        resource = OSUtils.checkResource(configuration,false);
-        Assert.assertTrue(resource);
-
-        configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,0);
-        configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,Double.MAX_VALUE);
-        resource = OSUtils.checkResource(configuration,false);
-        Assert.assertFalse(resource);
-
     }
 
 }

dolphinscheduler-dist/release-docs/LICENSE

@@ -471,7 +471,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
     aspectjweaver 1.9.2:https://mvnrepository.com/artifact/org.aspectj/aspectjweaver/1.9.2, EPL 1.0
     logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1
     logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1
-    oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0
+    oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0
     junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0
     h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0
 

pom.xml

@@ -92,7 +92,7 @@
         <postgresql.version>42.1.4</postgresql.version>
         <hive.jdbc.version>2.1.0</hive.jdbc.version>
         <commons.io.version>2.4</commons.io.version>
-        <oshi.core.version>3.5.0</oshi.core.version>
+        <oshi.core.version>3.9.1</oshi.core.version>
         <clickhouse.jdbc.version>0.1.52</clickhouse.jdbc.version>
         <mssql.jdbc.version>6.1.0.jre8</mssql.jdbc.version>
         <presto.jdbc.version>0.238.1</presto.jdbc.version>

tools/dependencies/known-dependencies.txt

@@ -179,7 +179,7 @@ mybatis-spring-2.0.2.jar
 netty-3.6.2.Final.jar
 netty-all-4.1.52.Final.jar
 opencsv-2.3.jar
-oshi-core-3.5.0.jar
+oshi-core-3.9.1.jar
 paranamer-2.3.jar
 parquet-hadoop-bundle-1.8.1.jar
 plexus-cipher-1.7.jar
@@ -237,7 +237,6 @@ swagger-annotations-1.5.20.jar
 swagger-bootstrap-ui-1.9.3.jar
 swagger-models-1.5.24.jar
 tephra-api-0.6.0.jar
-threetenbp-1.3.6.jar
 transaction-api-1.1.jar
 validation-api-2.0.1.Final.jar
 wagon-provider-api-2.2.jar