From 7445fa946bfcaf2c2da7c0bd1536ac979bf1d240 Mon Sep 17 00:00:00 2001 From: dailidong Date: Fri, 19 Jun 2020 14:45:48 +0800 Subject: [PATCH] fix singe resource manager will NPE (#3013) * fix worker group config no effect * remove codehaus janino jar the license about janino maybe not compatiable with Apache v2 * Merge remote-tracking branch 'upstream/dev-1.3.0' into dev-1.3.0 # Conflicts: # dolphinscheduler-server/src/main/resources/config/install_config.conf * datasource config * Update datasource.properties * fix RunConfig bug * remove param monitor server state * fix table T_DS_ALERT * update h2 database * fix #2910 master server will show exception for some time when it restart * fix oom when no master is active * fix worker oom when master server restart * fix oom * fix * add UT * fix worker group config no effect * fix singe resource manager will NPE * Update HadoopUtilsTest.java --- .../common/utils/HadoopUtils.java | 20 +++++++++---------- .../src/main/resources/common.properties | 2 +- .../common/utils/HadoopUtilsTest.java | 4 ++-- .../main/resources/config/install_config.conf | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 963aff5f31..b4eebd6fc5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -186,7 +186,7 @@ public class HadoopUtils implements Closeable { * @param applicationId application id * @return url of application */ - public String getApplicationUrl(String applicationId) { + public String getApplicationUrl(String applicationId) throws Exception { /** * if rmHaIds contains xx, it signs not use resourcemanager * otherwise: @@ -194,21 +194,21 @@ public class HadoopUtils implements Closeable { * if rmHaIds not empty: resourcemanager HA enabled */ String appUrl = ""; - //not use resourcemanager - if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){ - yarnEnabled = false; - logger.warn("should not step here"); - } else if (!StringUtils.isEmpty(rmHaIds)) { + if (StringUtils.isEmpty(rmHaIds)){ + //single resourcemanager enabled + appUrl = appAddress; + yarnEnabled = true; + } else { //resourcemanager HA enabled appUrl = getAppAddress(appAddress, rmHaIds); yarnEnabled = true; logger.info("application url : {}", appUrl); - } else { - //single resourcemanager enabled - yarnEnabled = true; } + if(StringUtils.isBlank(appUrl)){ + throw new Exception("application url is blank"); + } return String.format(appUrl, applicationId); } @@ -402,7 +402,7 @@ public class HadoopUtils implements Closeable { * @return the return may be null or there may be other parse exceptions * @throws JSONException json exception */ - public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException { + public ExecutionStatus getApplicationStatus(String applicationId) throws Exception { if (StringUtils.isEmpty(applicationId)) { return null; } diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 923f02abd9..ef49a1962f 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -54,7 +54,7 @@ fs.s3a.access.key=A3DXS30FO22544RE # if resource.storage.type=S3，s3 secret key fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK -# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx # if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname. diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java index e239fe7cb0..a002c79c36 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java @@ -142,7 +142,7 @@ public class HadoopUtilsTest { @Test public void isYarnEnabled() { boolean result = hadoopUtils.isYarnEnabled(); - Assert.assertEquals(false, result); + Assert.assertEquals(true, result); } @Test @@ -185,7 +185,7 @@ public class HadoopUtilsTest { } @Test - public void getApplicationUrl(){ + public void getApplicationUrl() throws Exception { String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167"); logger.info(application_1516778421218_0042); } diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf index 05a3e9906c..eeff0308b1 100644 --- a/dolphinscheduler-server/src/main/resources/config/install_config.conf +++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf @@ -87,7 +87,7 @@ s3Endpoint="http://192.168.xx.xx:9010" s3AccessKey="xxxxxxxxxx" s3SecretKey="xxxxxxxxxx" -# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty +# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty yarnHaIps="192.168.xx.xx,192.168.xx.xx" # if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.