github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[Fix-5309][Common] Fix the wrong memory (#5312) 

* [Fix-5309][Common] Fix the wrong memory

* [Fix-5309][Common] Fix unit test and remove unused code
pull/3/MERGE
Shiwen Cheng 4 years ago committed by GitHub
parent
5ef1f731b7
commit
e92e29ef9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 32 additions and 106 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  2. 35
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
  3. 46
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
  4. 40
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
  5. 2
      dolphinscheduler-dist/release-docs/LICENSE
  6. 2
      pom.xml
  7. 3
      tools/dependencies/known-dependencies.txt

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
Unescape View File

@ -380,21 +380,11 @@ public final class Constants {
*/
public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
/**
* master reserved memory
*/
public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
/**
* worker cpu load
*/
public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
/**
* worker reserved memory
*/
public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
/**
* worker host weight
*/

35
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
Unescape View File

@ -17,11 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.configuration.Configuration;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
@ -85,7 +82,7 @@ public class OSUtils {
*/
public static double memoryUsage() {
GlobalMemory memory = hal.getMemory();
double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10;
double memoryUsage = (memory.getTotal() - memory.getAvailable()) * 1.0 / memory.getTotal();
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
@ -101,12 +98,11 @@ public class OSUtils {
*/
public static double availablePhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024;
double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
}
/**
@ -116,13 +112,13 @@ public class OSUtils {
*
* @return available Physical Memory Size, unit: G
*/
public static double totalMemorySize() {
public static double totalPhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
return Double.parseDouble(df.format(totalPhysicalMemorySize));
}
/**
@ -504,25 +500,4 @@ public class OSUtils {
}
}
/**
* check memory and cpu usage
*
* @param conf conf
* @param isMaster is master
* @return check memory and cpu usage
*/
public static Boolean checkResource(Configuration conf, Boolean isMaster) {
double systemCpuLoad;
double systemReservedMemory;
if (Boolean.TRUE.equals(isMaster)) {
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY);
} else {
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY);
}
return checkResource(systemCpuLoad, systemReservedMemory);
}
}

46
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
Unescape View File

@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.os;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import oshi.hardware.GlobalMemory;
import java.math.RoundingMode;
import java.text.DecimalFormat;
/**
@ -34,42 +32,34 @@ public class OSUtilsTest {
private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class);
@Test
public void memoryUsage() {
logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239
double memoryUsage = OSUtils.memoryUsage();
logger.info("memoryUsage : {}", memoryUsage);
Assert.assertTrue(memoryUsage >= 0.0);
}
@Test
public void availablePhysicalMemorySize() {
logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize());
logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10);
public void physicalMemorySize() {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize);
logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize);
Assert.assertTrue(availablePhysicalMemorySize >= 0.0);
Assert.assertTrue(totalPhysicalMemorySize >= 0.0);
}
@Test
public void loadAverage() {
logger.info("memoryUsage : {}", OSUtils.loadAverage());
}
private void printMemory(GlobalMemory memory) {
logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() );
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage : {}", loadAverage);
Assert.assertTrue(loadAverage >= 0.0);
}
@Test
public void cpuUsage() throws Exception {
logger.info("cpuUsage : {}", OSUtils.cpuUsage());
Thread.sleep(1000L);
logger.info("cpuUsage : {}", OSUtils.cpuUsage());
public void cpuUsage() {
double cpuUsage = OSUtils.cpuUsage();
DecimalFormat df = new DecimalFormat("0.00");
df.setRoundingMode(RoundingMode.HALF_UP);
logger.info("cpuUsage1 : {}", df.format(cpuUsage));
logger.info("cpuUsage : {}", cpuUsage);
Assert.assertTrue(cpuUsage >= 0.0);
}
}

40
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
Unescape View File

@ -16,11 +16,6 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import java.io.IOException;
import java.util.List;
@ -40,12 +35,12 @@ public class OSUtilsTest {
}
@Test
public void testOSMetric(){
public void testOSMetric() {
if (!OSUtils.isWindows()) {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize >= 0.0d);
double totalMemorySize = OSUtils.totalMemorySize();
Assert.assertTrue(totalMemorySize >= 0.0d);
double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
Assert.assertTrue(totalPhysicalMemorySize >= 0.0d);
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage {}", loadAverage);
double memoryUsage = OSUtils.memoryUsage();
@ -92,7 +87,7 @@ public class OSUtilsTest {
@Test
public void exeCmd() {
if(OSUtils.isMacOS() || !OSUtils.isWindows()){
if (OSUtils.isMacOS() || !OSUtils.isWindows()) {
try {
String result = OSUtils.exeCmd("echo helloWorld");
Assert.assertEquals("helloWorld\n",result);
@ -102,39 +97,16 @@ public class OSUtilsTest {
}
}
@Test
public void getProcessID(){
public void getProcessID() {
int processId = OSUtils.getProcessID();
Assert.assertNotEquals(0, processId);
}
@Test
public void checkResource(){
public void checkResource() {
boolean resource = OSUtils.checkResource(100,0);
Assert.assertTrue(resource);
resource = OSUtils.checkResource(0,Double.MAX_VALUE);
Assert.assertFalse(resource);
Configuration configuration = new PropertiesConfiguration();
configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,100);
configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,0);
resource = OSUtils.checkResource(configuration,true);
Assert.assertTrue(resource);
configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,0);
configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,Double.MAX_VALUE);
resource = OSUtils.checkResource(configuration,true);
Assert.assertFalse(resource);
configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,100);
configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,0);
resource = OSUtils.checkResource(configuration,false);
Assert.assertTrue(resource);
configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,0);
configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,Double.MAX_VALUE);
resource = OSUtils.checkResource(configuration,false);
Assert.assertFalse(resource);
}
}

2
dolphinscheduler-dist/release-docs/LICENSE vendored
Unescape View File

@ -471,7 +471,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
aspectjweaver 1.9.2:https://mvnrepository.com/artifact/org.aspectj/aspectjweaver/1.9.2, EPL 1.0
logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1
logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1
oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0
oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0
junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0
h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0

2
pom.xml
Unescape View File

@ -92,7 +92,7 @@
<postgresql.version>42.1.4</postgresql.version>
<hive.jdbc.version>2.1.0</hive.jdbc.version>
<commons.io.version>2.4</commons.io.version>
<oshi.core.version>3.5.0</oshi.core.version>
<oshi.core.version>3.9.1</oshi.core.version>
<clickhouse.jdbc.version>0.1.52</clickhouse.jdbc.version>
<mssql.jdbc.version>6.1.0.jre8</mssql.jdbc.version>
<presto.jdbc.version>0.238.1</presto.jdbc.version>

3
tools/dependencies/known-dependencies.txt
Unescape View File

@ -179,7 +179,7 @@ mybatis-spring-2.0.2.jar
netty-3.6.2.Final.jar
netty-all-4.1.52.Final.jar
opencsv-2.3.jar
oshi-core-3.5.0.jar
oshi-core-3.9.1.jar
paranamer-2.3.jar
parquet-hadoop-bundle-1.8.1.jar
plexus-cipher-1.7.jar
@ -237,7 +237,6 @@ swagger-annotations-1.5.20.jar
swagger-bootstrap-ui-1.9.3.jar
swagger-models-1.5.24.jar
tephra-api-0.6.0.jar
threetenbp-1.3.6.jar
transaction-api-1.1.jar
validation-api-2.0.1.Final.jar
wagon-provider-api-2.2.jar

Write Preview
Loading…
Cancel
Save