Browse Source

fix singe resource manager will NPE (#3013)

* fix worker group config no effect

* remove codehaus janino jar
the license about janino maybe not compatiable with Apache v2

* Merge remote-tracking branch 'upstream/dev-1.3.0' into dev-1.3.0

# Conflicts:
#	dolphinscheduler-server/src/main/resources/config/install_config.conf

* datasource config

* Update datasource.properties

* fix RunConfig bug

* remove param monitor server state

* fix table T_DS_ALERT

* update h2 database

* fix #2910 master server will show exception for some time when it restart

* fix oom when no master is active

* fix worker oom when master server restart

* fix oom

* fix

* add UT

* fix worker group config no effect

* fix singe resource manager will NPE

* Update HadoopUtilsTest.java
pull/3/MERGE
dailidong 5 years ago committed by GitHub
parent
commit
620ecffa72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
  2. 2
      dolphinscheduler-common/src/main/resources/common.properties
  3. 4
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
  4. 2
      dolphinscheduler-server/src/main/resources/config/install_config.conf

20
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -186,7 +186,7 @@ public class HadoopUtils implements Closeable {
* @param applicationId application id
* @return url of application
*/
public String getApplicationUrl(String applicationId) {
public String getApplicationUrl(String applicationId) throws Exception {
/**
* if rmHaIds contains xx, it signs not use resourcemanager
* otherwise:
@ -194,21 +194,21 @@ public class HadoopUtils implements Closeable {
* if rmHaIds not empty: resourcemanager HA enabled
*/
String appUrl = "";
//not use resourcemanager
if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){
yarnEnabled = false;
logger.warn("should not step here");
} else if (!StringUtils.isEmpty(rmHaIds)) {
if (StringUtils.isEmpty(rmHaIds)){
//single resourcemanager enabled
appUrl = appAddress;
yarnEnabled = true;
} else {
//resourcemanager HA enabled
appUrl = getAppAddress(appAddress, rmHaIds);
yarnEnabled = true;
logger.info("application url : {}", appUrl);
} else {
//single resourcemanager enabled
yarnEnabled = true;
}
if(StringUtils.isBlank(appUrl)){
throw new Exception("application url is blank");
}
return String.format(appUrl, applicationId);
}
@ -402,7 +402,7 @@ public class HadoopUtils implements Closeable {
* @return the return may be null or there may be other parse exceptions
* @throws JSONException json exception
*/
public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException {
public ExecutionStatus getApplicationStatus(String applicationId) throws Exception {
if (StringUtils.isEmpty(applicationId)) {
return null;
}

2
dolphinscheduler-common/src/main/resources/common.properties

@ -54,7 +54,7 @@ fs.s3a.access.key=A3DXS30FO22544RE
# if resource.storage.type=S3,s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.

4
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java

@ -142,7 +142,7 @@ public class HadoopUtilsTest {
@Test
public void isYarnEnabled() {
boolean result = hadoopUtils.isYarnEnabled();
Assert.assertEquals(false, result);
Assert.assertEquals(true, result);
}
@Test
@ -185,7 +185,7 @@ public class HadoopUtilsTest {
}
@Test
public void getApplicationUrl(){
public void getApplicationUrl() throws Exception {
String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167");
logger.info(application_1516778421218_0042);
}

2
dolphinscheduler-server/src/main/resources/config/install_config.conf

@ -87,7 +87,7 @@ s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarnHaIps="192.168.xx.xx,192.168.xx.xx"
# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.

Loading…
Cancel
Save