|
|
@ -22,6 +22,8 @@ import org.apache.hadoop.conf.Configuration; |
|
|
|
import org.junit.Assert; |
|
|
|
import org.junit.Assert; |
|
|
|
import org.junit.Ignore; |
|
|
|
import org.junit.Ignore; |
|
|
|
import org.junit.Test; |
|
|
|
import org.junit.Test; |
|
|
|
|
|
|
|
import org.junit.runner.RunWith; |
|
|
|
|
|
|
|
import org.mockito.junit.MockitoJUnitRunner; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
|
|
|
|
|
|
|
@ -29,91 +31,93 @@ import java.io.IOException; |
|
|
|
import java.util.Arrays; |
|
|
|
import java.util.Arrays; |
|
|
|
import java.util.List; |
|
|
|
import java.util.List; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@RunWith(MockitoJUnitRunner.class) |
|
|
|
//todo there is no hadoop environment
|
|
|
|
//todo there is no hadoop environment
|
|
|
|
public class HadoopUtilsTest { |
|
|
|
public class HadoopUtilsTest { |
|
|
|
|
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class); |
|
|
|
private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class); |
|
|
|
|
|
|
|
private HadoopUtils hadoopUtils = HadoopUtils.getInstance(); |
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getActiveRMTest() { |
|
|
|
public void getActiveRMTest() { |
|
|
|
logger.info(HadoopUtils.getAppAddress("http://ark1:8088/ws/v1/cluster/apps/%s","192.168.xx.xx,192.168.xx.xx")); |
|
|
|
logger.info(hadoopUtils.getAppAddress("http://ark1:8088/ws/v1/cluster/apps/%s","192.168.xx.xx,192.168.xx.xx")); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getApplicationStatusAddressTest(){ |
|
|
|
public void rename() throws IOException { |
|
|
|
logger.info(HadoopUtils.getInstance().getApplicationUrl("application_1548381297012_0030")); |
|
|
|
boolean result = hadoopUtils.rename("/dolphinscheduler/hdfs1","/dolphinscheduler/hdfs2"); |
|
|
|
|
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getConfiguration(){ |
|
|
|
public void getConfiguration(){ |
|
|
|
Configuration conf = HadoopUtils.getInstance().getConfiguration(); |
|
|
|
Configuration conf = hadoopUtils.getConfiguration(); |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void mkdir() throws IOException { |
|
|
|
public void mkdir() throws IOException { |
|
|
|
boolean result = HadoopUtils.getInstance().mkdir("/dolphinscheduler/hdfs"); |
|
|
|
boolean result = hadoopUtils.mkdir("/dolphinscheduler/hdfs"); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void delete() throws IOException { |
|
|
|
public void delete() throws IOException { |
|
|
|
boolean result = HadoopUtils.getInstance().delete("/dolphinscheduler/hdfs",true); |
|
|
|
boolean result = hadoopUtils.delete("/dolphinscheduler/hdfs",true); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void exists() throws IOException { |
|
|
|
public void exists() throws IOException { |
|
|
|
boolean result = HadoopUtils.getInstance().exists("/dolphinscheduler/hdfs"); |
|
|
|
boolean result = hadoopUtils.exists("/dolphinscheduler/hdfs"); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
Assert.assertEquals(true, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsDataBasePath() throws IOException { |
|
|
|
public void getHdfsDataBasePath() throws IOException { |
|
|
|
String result = HadoopUtils.getInstance().getHdfsDataBasePath(); |
|
|
|
String result = hadoopUtils.getHdfsDataBasePath(); |
|
|
|
Assert.assertEquals("/dolphinscheduler", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsResDir() throws IOException { |
|
|
|
public void getHdfsResDir() throws IOException { |
|
|
|
String result = HadoopUtils.getInstance().getHdfsResDir("11000"); |
|
|
|
String result = hadoopUtils.getHdfsResDir("11000"); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsUserDir() throws IOException { |
|
|
|
public void getHdfsUserDir() throws IOException { |
|
|
|
String result = HadoopUtils.getInstance().getHdfsUserDir("11000",1000); |
|
|
|
String result = hadoopUtils.getHdfsUserDir("11000",1000); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/home/1000", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/home/1000", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsUdfDir() throws IOException { |
|
|
|
public void getHdfsUdfDir() throws IOException { |
|
|
|
String result = HadoopUtils.getInstance().getHdfsUdfDir("11000"); |
|
|
|
String result = hadoopUtils.getHdfsUdfDir("11000"); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/udfs", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/11000/udfs", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getHdfsFileName() throws IOException { |
|
|
|
public void getHdfsFileName() throws IOException { |
|
|
|
String result = HadoopUtils.getInstance().getHdfsFileName(ResourceType.FILE,"11000","aa.txt"); |
|
|
|
String result = hadoopUtils.getHdfsFileName(ResourceType.FILE,"11000","aa.txt"); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000/aa.txt", result); |
|
|
|
Assert.assertEquals("/dolphinscheduler/resources/11000/aa.txt", result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void isYarnEnabled() throws IOException { |
|
|
|
public void isYarnEnabled() throws IOException { |
|
|
|
boolean result = HadoopUtils.getInstance().isYarnEnabled(); |
|
|
|
boolean result = hadoopUtils.isYarnEnabled(); |
|
|
|
Assert.assertEquals(false, result); |
|
|
|
Assert.assertEquals(false, result); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void test() throws IOException { |
|
|
|
public void test() throws IOException { |
|
|
|
HadoopUtils.getInstance().copyLocalToHdfs("/root/teamviewer_13.1.8286.x86_64.rpm", "/journey", true, true); |
|
|
|
hadoopUtils.copyLocalToHdfs("/root/teamviewer_13.1.8286.x86_64.rpm", "/journey", true, true); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void readFileTest(){ |
|
|
|
public void readFileTest(){ |
|
|
|
try { |
|
|
|
try { |
|
|
|
byte[] bytes = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/35435.sh"); |
|
|
|
byte[] bytes = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/35435.sh"); |
|
|
|
logger.info(new String(bytes)); |
|
|
|
logger.info(new String(bytes)); |
|
|
|
} catch (Exception e) { |
|
|
|
} catch (Exception e) { |
|
|
|
logger.error(e.getMessage(),e); |
|
|
|
logger.error(e.getMessage(),e); |
|
|
@ -123,9 +127,8 @@ public class HadoopUtilsTest { |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void testMove(){ |
|
|
|
public void testMove(){ |
|
|
|
HadoopUtils instance = HadoopUtils.getInstance(); |
|
|
|
|
|
|
|
try { |
|
|
|
try { |
|
|
|
instance.copy("/opt/apptest/test.dat","/opt/apptest/test.dat.back",true,true); |
|
|
|
hadoopUtils.copy("/opt/apptest/test.dat","/opt/apptest/test.dat.back",true,true); |
|
|
|
} catch (Exception e) { |
|
|
|
} catch (Exception e) { |
|
|
|
logger.error(e.getMessage(), e); |
|
|
|
logger.error(e.getMessage(), e); |
|
|
|
} |
|
|
|
} |
|
|
@ -134,24 +137,24 @@ public class HadoopUtilsTest { |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getApplicationStatus() { |
|
|
|
public void getApplicationStatus() { |
|
|
|
logger.info(HadoopUtils.getInstance().getApplicationStatus("application_1542010131334_0029").toString()); |
|
|
|
logger.info(hadoopUtils.getApplicationStatus("application_1542010131334_0029").toString()); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void getApplicationUrl(){ |
|
|
|
public void getApplicationUrl(){ |
|
|
|
String application_1516778421218_0042 = HadoopUtils.getInstance().getApplicationUrl("application_1529051418016_0167"); |
|
|
|
String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167"); |
|
|
|
logger.info(application_1516778421218_0042); |
|
|
|
logger.info(application_1516778421218_0042); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void catFileWithLimitTest()throws Exception{ |
|
|
|
public void catFileWithLimitTest()throws Exception{ |
|
|
|
List<String> stringList = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); |
|
|
|
List<String> stringList = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); |
|
|
|
logger.info(String.join(",",stringList)); |
|
|
|
logger.info(String.join(",",stringList)); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@Test |
|
|
|
@Test |
|
|
|
public void catFileTest()throws Exception{ |
|
|
|
public void catFileTest()throws Exception{ |
|
|
|
byte[] content = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py"); |
|
|
|
byte[] content = hadoopUtils.catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py"); |
|
|
|
logger.info(Arrays.toString(content)); |
|
|
|
logger.info(Arrays.toString(content)); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|