diff --git a/escheduler-common/pom.xml b/escheduler-common/pom.xml
index dc95fc7f9c..4f76b55cac 100644
--- a/escheduler-common/pom.xml
+++ b/escheduler-common/pom.xml
@@ -230,6 +230,29 @@
+
+ org.apache.hadoop
+ hadoop-aws
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+
+
+
+
org.apache.commons
commons-lang3
diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java
index f114b112a5..b76d6347da 100644
--- a/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java
+++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/HadoopUtils.java
@@ -34,12 +34,14 @@ import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static cn.escheduler.common.Constants.*;
import static cn.escheduler.common.utils.PropertyUtils.getInt;
import static cn.escheduler.common.utils.PropertyUtils.getString;
+import static cn.escheduler.common.utils.PropertyUtils.getPrefixedProperties;
/**
* hadoop utils
@@ -76,7 +78,9 @@ public class HadoopUtils implements Closeable {
if(defaultFS.startsWith("file")){
String defaultFSProp = getString(FS_DEFAULTFS);
if(StringUtils.isNotBlank(defaultFSProp)){
+ Map fsRelatedProps = getPrefixedProperties("fs.");
configuration.set(FS_DEFAULTFS,defaultFSProp);
+ fsRelatedProps.entrySet().stream().forEach(entry -> configuration.set(entry.getKey(), entry.getValue()));
}else{
logger.error("property:{} can not to be empty, please set!");
throw new RuntimeException("property:{} can not to be empty, please set!");
@@ -316,7 +320,13 @@ public class HadoopUtils implements Closeable {
* @return data hdfs path
*/
public static String getHdfsDataBasePath() {
- return getString(DATA_STORE_2_HDFS_BASEPATH);
+ String basePath = getString(DATA_STORE_2_HDFS_BASEPATH);
+ if ("/".equals(basePath)) {
+ // if basepath is configured to /, the generated url may be //default/resources (with extra leading /)
+ return "";
+ } else {
+ return basePath;
+ }
}
/**
@@ -365,7 +375,7 @@ public class HadoopUtils implements Closeable {
* @return file directory of tenants on hdfs
*/
private static String getHdfsTenantDir(String tenantCode) {
- return String.format("%s/%s", getString(DATA_STORE_2_HDFS_BASEPATH), tenantCode);
+ return String.format("%s/%s", getHdfsDataBasePath(), tenantCode);
}
diff --git a/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java b/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java
index 363bb417a8..f5dab12618 100644
--- a/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java
+++ b/escheduler-common/src/main/java/cn/escheduler/common/utils/PropertyUtils.java
@@ -22,6 +22,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import static cn.escheduler.common.Constants.COMMON_PROPERTIES_PATH;
@@ -189,4 +191,19 @@ public class PropertyUtils {
String val = getString(key);
return val == null ? defaultValue : Enum.valueOf(type, val);
}
+
+ /**
+ * get all properties with specified prefix, like: fs.
+ * @param prefix prefix to search
+ * @return
+ */
+ public static Map getPrefixedProperties(String prefix) {
+ Map matchedProperties = new HashMap<>();
+ for (String propName : properties.stringPropertyNames()) {
+ if (propName.startsWith(prefix)) {
+ matchedProperties.put(propName, properties.getProperty(propName));
+ }
+ }
+ return matchedProperties;
+ }
}
diff --git a/pom.xml b/pom.xml
index 3e5ff0d79d..15849c8382 100644
--- a/pom.xml
+++ b/pom.xml
@@ -323,6 +323,11 @@
hadoop-yarn-common
${hadoop.version}
+
+ org.apache.hadoop
+ hadoop-aws
+ ${hadoop.version}
+
javax.servlet