|
|
@ -37,14 +37,25 @@ public class HadoopUtilsTest { |
|
|
|
|
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class); |
|
|
|
private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class); |
|
|
|
private HadoopUtils hadoopUtils = HadoopUtils.getInstance(); |
|
|
|
private HadoopUtils hadoopUtils = HadoopUtils.getInstance(); |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getActiveRMTest() { |
|
|
|
public void getActiveRMTest() { |
|
|
|
logger.info(hadoopUtils.getAppAddress("http://ark1:8088/ws/v1/cluster/apps/%s","192.168.xx.xx,192.168.xx.xx")); |
|
|
|
try{ |
|
|
|
|
|
|
|
hadoopUtils.getAppAddress("http://ark1:8088/ws/v1/cluster/apps/%s","192.168.xx.xx,192.168.xx.xx"); |
|
|
|
|
|
|
|
} catch (Exception e) { |
|
|
|
|
|
|
|
logger.error(e.getMessage(),e); |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void rename() throws IOException { |
|
|
|
public void rename() { |
|
|
|
boolean result = hadoopUtils.rename("/dolphinscheduler/hdfs1","/dolphinscheduler/hdfs2"); |
|
|
|
|
|
|
|
|
|
|
|
boolean result = false; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
result = hadoopUtils.rename("/dolphinscheduler/hdfs1","/dolphinscheduler/hdfs2"); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
logger.error(e.getMessage(),e); |
|
|
|
|
|
|
|
} |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
@ -56,62 +67,81 @@ public class HadoopUtilsTest { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void mkdir() throws IOException { |
|
|
|
public void mkdir() { |
|
|
|
boolean result = hadoopUtils.mkdir("/dolphinscheduler/hdfs"); |
|
|
|
boolean result = false; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
result = hadoopUtils.mkdir("/dolphinscheduler/hdfs"); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
e.printStackTrace(); |
|
|
|
|
|
|
|
} |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void delete() throws IOException { |
|
|
|
public void delete() { |
|
|
|
boolean result = hadoopUtils.delete("/dolphinscheduler/hdfs",true); |
|
|
|
boolean result = false; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
result = hadoopUtils.delete("/dolphinscheduler/hdfs",true); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
e.printStackTrace(); |
|
|
|
|
|
|
|
} |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void exists() throws IOException { |
|
|
|
public void exists() { |
|
|
|
boolean result = hadoopUtils.exists("/dolphinscheduler/hdfs"); |
|
|
|
boolean result = false; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
result = hadoopUtils.exists("/dolphinscheduler/hdfs"); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
e.printStackTrace(); |
|
|
|
|
|
|
|
} |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsDataBasePath() throws IOException { |
|
|
|
public void getHdfsDataBasePath() { |
|
|
|
String result = hadoopUtils.getHdfsDataBasePath(); |
|
|
|
String result = hadoopUtils.getHdfsDataBasePath(); |
|
|
|
Assert.assertEquals("/dolphinscheduler", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsResDir() throws IOException { |
|
|
|
public void getHdfsResDir() { |
|
|
|
String result = hadoopUtils.getHdfsResDir("11000"); |
|
|
|
String result = hadoopUtils.getHdfsResDir("11000"); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsUserDir() throws IOException { |
|
|
|
public void getHdfsUserDir() { |
|
|
|
String result = hadoopUtils.getHdfsUserDir("11000",1000); |
|
|
|
String result = hadoopUtils.getHdfsUserDir("11000",1000); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/home/1000", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/home/1000", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsUdfDir() throws IOException { |
|
|
|
public void getHdfsUdfDir() { |
|
|
|
String result = hadoopUtils.getHdfsUdfDir("11000"); |
|
|
|
String result = hadoopUtils.getHdfsUdfDir("11000"); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/udfs", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/udfs", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsFileName() throws IOException { |
|
|
|
public void getHdfsFileName() { |
|
|
|
String result = hadoopUtils.getHdfsFileName(ResourceType.FILE,"11000","aa.txt"); |
|
|
|
String result = hadoopUtils.getHdfsFileName(ResourceType.FILE,"11000","aa.txt"); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000/aa.txt", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000/aa.txt", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void isYarnEnabled() throws IOException { |
|
|
|
public void isYarnEnabled() { |
|
|
|
boolean result = hadoopUtils.isYarnEnabled(); |
|
|
|
boolean result = hadoopUtils.isYarnEnabled(); |
|
|
|
Assert.assertEquals(false, result); |
|
|
|
Assert.assertEquals(false, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void test() throws IOException { |
|
|
|
public void test() { |
|
|
|
|
|
|
|
try { |
|
|
|
hadoopUtils.copyLocalToHdfs("/root/teamviewer_13.1.8286.x86_64.rpm", "/journey", true, true); |
|
|
|
hadoopUtils.copyLocalToHdfs("/root/teamviewer_13.1.8286.x86_64.rpm", "/journey", true, true); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
e.printStackTrace(); |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
@ -147,14 +177,24 @@ public class HadoopUtilsTest { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void catFileWithLimitTest()throws Exception{ |
|
|
|
public void catFileWithLimitTest() { |
|
|
|
List<String> stringList = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); |
|
|
|
List<String> stringList = null; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
stringList = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
e.printStackTrace(); |
|
|
|
|
|
|
|
} |
|
|
|
logger.info(String.join(",",stringList)); |
|
|
|
logger.info(String.join(",",stringList)); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void catFileTest()throws Exception{ |
|
|
|
public void catFileTest() { |
|
|
|
byte[] content = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py"); |
|
|
|
byte[] content = new byte[0]; |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
content = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py"); |
|
|
|
|
|
|
|
} catch (IOException e) { |
|
|
|
|
|
|
|
e.printStackTrace(); |
|
|
|
|
|
|
|
} |
|
|
|
logger.info(Arrays.toString(content)); |
|
|
|
logger.info(Arrays.toString(content)); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|