Browse Source

fix hdfs defaultFs not working (#11823)

3.2.0-release
xiangzihao 2 years ago committed by GitHub
parent
commit
ebcffb04aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  2. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -65,6 +65,11 @@ public final class Constants {
*/ */
public static final String FS_DEFAULT_FS = "resource.hdfs.fs.defaultFS"; public static final String FS_DEFAULT_FS = "resource.hdfs.fs.defaultFS";
/**
* hdfs defaultFS property name. Should be consistent with the property name in hdfs-site.xml
*/
public static final String HDFS_DEFAULT_FS = "fs.defaultFS";
/** /**
* hadoop configuration * hadoop configuration
*/ */

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@ -70,7 +70,7 @@ import com.google.common.cache.LoadingCache;
public class HadoopUtils implements Closeable, StorageOperate { public class HadoopUtils implements Closeable, StorageOperate {
private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class); private static final Logger logger = LoggerFactory.getLogger(HadoopUtils.class);
private String hdfsUser = PropertyUtils.getString(Constants.HDFS_ROOT_USER); private String hdfsUser;
public static final String RM_HA_IDS = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS); public static final String RM_HA_IDS = PropertyUtils.getString(Constants.YARN_RESOURCEMANAGER_HA_RM_IDS);
public static final String APP_ADDRESS = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS); public static final String APP_ADDRESS = PropertyUtils.getString(Constants.YARN_APPLICATION_STATUS_ADDRESS);
public static final String JOB_HISTORY_ADDRESS = PropertyUtils.getString(Constants.YARN_JOB_HISTORY_STATUS_ADDRESS); public static final String JOB_HISTORY_ADDRESS = PropertyUtils.getString(Constants.YARN_JOB_HISTORY_STATUS_ADDRESS);
@ -140,7 +140,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
// the default is the local file system // the default is the local file system
if (StringUtils.isNotBlank(defaultFS)) { if (StringUtils.isNotBlank(defaultFS)) {
Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("fs."); Map<String, String> fsRelatedProps = PropertyUtils.getPrefixedProperties("fs.");
configuration.set(Constants.FS_DEFAULT_FS, defaultFS); configuration.set(Constants.HDFS_DEFAULT_FS, defaultFS);
fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); fsRelatedProps.forEach((key, value) -> configuration.set(key, value));
} else { } else {
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULT_FS); logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULT_FS);

Loading…
Cancel
Save