|
|
@ -119,7 +119,9 @@ public class HadoopUtils implements Closeable { |
|
|
|
fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); |
|
|
|
fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); |
|
|
|
}else{ |
|
|
|
}else{ |
|
|
|
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS ); |
|
|
|
logger.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULTFS ); |
|
|
|
throw new RuntimeException("property:{} can not to be empty, please set!"); |
|
|
|
throw new RuntimeException( |
|
|
|
|
|
|
|
String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULTFS) |
|
|
|
|
|
|
|
); |
|
|
|
} |
|
|
|
} |
|
|
|
}else{ |
|
|
|
}else{ |
|
|
|
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); |
|
|
|
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS); |
|
|
@ -219,12 +221,14 @@ public class HadoopUtils implements Closeable { |
|
|
|
return null; |
|
|
|
return null; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
FSDataInputStream in = fs.open(new Path(hdfsFilePath)); |
|
|
|
try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))){ |
|
|
|
BufferedReader br = new BufferedReader(new InputStreamReader(in)); |
|
|
|
BufferedReader br = new BufferedReader(new InputStreamReader(in)); |
|
|
|
Stream<String> stream = br.lines().skip(skipLineNums).limit(limit); |
|
|
|
Stream<String> stream = br.lines().skip(skipLineNums).limit(limit); |
|
|
|
return stream.collect(Collectors.toList()); |
|
|
|
return stream.collect(Collectors.toList()); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* make the given file and all non-existent parents into |
|
|
|
* make the given file and all non-existent parents into |
|
|
|
* directories. Has the semantics of Unix 'mkdir -p'. |
|
|
|
* directories. Has the semantics of Unix 'mkdir -p'. |
|
|
|