From bb13f2eae78e4b829816068e8ca3d337d97a0fd2 Mon Sep 17 00:00:00 2001 From: BoYiZhang <39816903+BoYiZhang@users.noreply.github.com> Date: Fri, 27 Nov 2020 21:55:28 +0800 Subject: [PATCH] [Bug][Common] read file garbled (#3479) * fix bug : Random code problem Co-authored-by: zhanglong --- .../api/service/ResourcesServiceTest.java | 308 +++++++++--------- .../common/utils/FileUtils.java | 5 +- .../common/utils/HadoopUtils.java | 3 +- .../common/utils/FileUtilsTest.java | 20 +- 4 files changed, 185 insertions(+), 151 deletions(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 14cd394611..d430d3a755 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -14,12 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -33,7 +31,18 @@ import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -49,14 +58,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockMultipartFile; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"sun.security.*", "javax.net.*"}) -@PrepareForTest({HadoopUtils.class,PropertyUtils.class, FileUtils.class,org.apache.dolphinscheduler.api.utils.FileUtils.class}) +@PrepareForTest({HadoopUtils.class, PropertyUtils.class, FileUtils.class, org.apache.dolphinscheduler.api.utils.FileUtils.class}) public class ResourcesServiceTest { private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class); @@ -94,49 +101,48 @@ public class ResourcesServiceTest { } @Test - public void testCreateResource(){ + public void testCreateResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null,-1,"/"); + Result result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_FILE_IS_EMPTY - MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes()); + MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.createResource(user,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/"); + result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_FORBID_CHANGE - mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes()); + mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()); PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,mockMultipartFile,-1,"/"); + result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg()); //UDF_RESOURCE_SUFFIX_NOT_JAR - mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes()); + mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes()); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf"); - result = resourcesService.createResource(user,"ResourcesServiceTest.pdf","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile,-1,"/"); + result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest", ResourceType.UDF, mockMultipartFile, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg()); } @Test - public void testCreateDirecotry(){ + public void testCreateDirecotry() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/"); + Result result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //PARENT_RESOURCE_NOT_EXIST user.setId(1); @@ -145,53 +151,53 @@ public class ResourcesServiceTest { Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,1,"/"); + result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, 1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.queryResourceList("/directoryTest", 0, 0)).thenReturn(getResourceList()); - result = resourcesService.createDirectory(user,"directoryTest","directory test",ResourceType.FILE,-1,"/"); + result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); } @Test - public void testUpdateResource(){ + public void testUpdateResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); //HDFS_NOT_STARTUP - Result result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); + Result result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.updateResource(user,0,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); + result = resourcesService.updateResource(user, 0, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //USER_NO_OPERATION_PERM - result = resourcesService.updateResource(user,1,"ResourcesServiceTest","ResourcesServiceTest",ResourceType.FILE,null); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(),result.getMsg()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST user.setId(1); Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(),Mockito.anyString())).thenReturn("test1"); + PowerMockito.when(HadoopUtils.getHdfsFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1"); try { Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(false); } catch (IOException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF,null); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //SUCCESS user.setId(1); @@ -200,57 +206,56 @@ public class ResourcesServiceTest { try { Mockito.when(HadoopUtils.getInstance().exists(Mockito.any())).thenReturn(true); } catch (IOException e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.FILE,null); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); //RESOURCE_EXIST Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest1.jar", 0, 0)).thenReturn(getResourceList()); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.FILE,null); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.FILE, null); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); //USER_NOT_EXIST Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF,null); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null); logger.info(result.toString()); Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF,null); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", ResourceType.UDF, null); logger.info(result.toString()); - Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); //SUCCESS Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test"); try { - PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(),Mockito.anyString(),true,true)).thenReturn(true); + PowerMockito.when(HadoopUtils.getInstance().copy(Mockito.anyString(), Mockito.anyString(), true, true)).thenReturn(true); } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF,null); + result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest1.jar", ResourceType.UDF, null); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testQueryResourceListPaging(){ + public void testQueryResourceListPaging() { User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); - IPage resourcePage = new Page<>(1,10); + IPage resourcePage = new Page<>(1, 10); resourcePage.setTotal(1); resourcePage.setRecords(getResourceList()); Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class), - Mockito.eq(0),Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); - Map result = resourcesService.queryResourceListPaging(loginUser,-1,ResourceType.FILE,"test",1,10); + Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); + Map result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); @@ -259,11 +264,11 @@ public class ResourcesServiceTest { } @Test - public void testQueryResourceList(){ + public void testQueryResourceList() { User loginUser = new User(); loginUser.setId(0); loginUser.setUserType(UserType.ADMIN_USER); - Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0,0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0, 0)).thenReturn(getResourceList()); Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); @@ -272,7 +277,7 @@ public class ResourcesServiceTest { } @Test - public void testDelete(){ + public void testDelete() { User loginUser = new User(); loginUser.setId(0); @@ -281,19 +286,19 @@ public class ResourcesServiceTest { try { // HDFS_NOT_STARTUP - Result result = resourcesService.delete(loginUser,1); + Result result = resourcesService.delete(loginUser, 1); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - result = resourcesService.delete(loginUser,2); + result = resourcesService.delete(loginUser, 2); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // USER_NO_OPERATION_PERM - result = resourcesService.delete(loginUser,2); + result = resourcesService.delete(loginUser, 2); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); @@ -301,30 +306,30 @@ public class ResourcesServiceTest { loginUser.setUserType(UserType.ADMIN_USER); loginUser.setTenantId(2); Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(loginUser); - result = resourcesService.delete(loginUser,1); + result = resourcesService.delete(loginUser, 1); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); //SUCCESS loginUser.setTenantId(1); Mockito.when(hadoopUtils.delete(Mockito.anyString(), Mockito.anyBoolean())).thenReturn(true); - result = resourcesService.delete(loginUser,1); + result = resourcesService.delete(loginUser, 1); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } catch (Exception e) { - logger.error("delete error",e); + logger.error("delete error", e); Assert.assertTrue(false); } } @Test - public void testVerifyResourceName(){ + public void testVerifyResourceName() { User user = new User(); user.setId(1); Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest.jar", 0, 0)).thenReturn(getResourceList()); - Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user); + Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); @@ -334,94 +339,89 @@ public class ResourcesServiceTest { try { Mockito.when(hadoopUtils.exists(unExistFullName)).thenReturn(false); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } - result = resourcesService.verifyResourceName("/test.jar",ResourceType.FILE,user); + result = resourcesService.verifyResourceName("/test.jar", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); - //RESOURCE_FILE_EXIST user.setTenantId(1); try { Mockito.when(hadoopUtils.exists("test")).thenReturn(true); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } PowerMockito.when(HadoopUtils.getHdfsResourceFileName("123", "test1")).thenReturn("test"); - result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user); + result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); - Assert.assertTrue(Status.RESOURCE_EXIST.getCode()==result.getCode()); + Assert.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode()); //SUCCESS - result = resourcesService.verifyResourceName("test2",ResourceType.FILE,user); + result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testReadResource(){ + public void testReadResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); //HDFS_NOT_STARTUP - Result result = resourcesService.readResource(1,1,10); + Result result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.readResource(2,1,10); + result = resourcesService.readResource(2, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); //USER_NOT_EXIST PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertTrue(Status.USER_NOT_EXIST.getCode()==result.getCode()); - + Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_FILE_NOT_EXIST Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); try { Mockito.when(hadoopUtils.exists(Mockito.anyString())).thenReturn(false); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode()==result.getCode()); + Assert.assertTrue(Status.RESOURCE_FILE_NOT_EXIST.getCode() == result.getCode()); //SUCCESS try { Mockito.when(hadoopUtils.exists(null)).thenReturn(true); - Mockito.when(hadoopUtils.catFile(null,1,10)).thenReturn(getContent()); + Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent()); } catch (IOException e) { - logger.error("hadoop error",e); + logger.error("hadoop error", e); } - result = resourcesService.readResource(1,1,10); + result = resourcesService.readResource(1, 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @@ -433,23 +433,23 @@ public class ResourcesServiceTest { PowerMockito.when(HadoopUtils.getHdfsUdfDir("udfDir")).thenReturn("udfDir"); User user = getUser(); //HDFS_NOT_STARTUP - Result result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); + Result result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); - result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); //RuntimeException try { PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",-1,"/"); - }catch (RuntimeException ex){ + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); + } catch (RuntimeException ex) { logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage()); } @@ -457,50 +457,48 @@ public class ResourcesServiceTest { //SUCCESS Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.onlineCreateResource(user,ResourceType.FILE,"test","jar","desc","content",-1,"/"); + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, "/"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); - + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testUpdateResourceContent(){ + public void testUpdateResourceContent() { User loginUser = new User(); loginUser.setId(0); PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); // HDFS_NOT_STARTUP - Result result = resourcesService.updateResourceContent(1,"content"); + Result result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); //RESOURCE_NOT_EXIST PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - result = resourcesService.updateResourceContent(2,"content"); + result = resourcesService.updateResourceContent(2, "content"); logger.info(result.toString()); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class"); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); - Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(),result.getMsg()); + Assert.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); //USER_NOT_EXIST PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar"); PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); - //TENANT_NOT_EXIST Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertTrue(Status.TENANT_NOT_EXIST.getCode() == result.getCode()); @@ -508,13 +506,13 @@ public class ResourcesServiceTest { Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.updateResourceContent(1,"content"); + result = resourcesService.updateResourceContent(1, "content"); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test - public void testDownloadResource(){ + public void testDownloadResource() { PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); @@ -530,102 +528,117 @@ public class ResourcesServiceTest { resource = resourcesService.downloadResource(1); Assert.assertNotNull(resource); } catch (Exception e) { - logger.error("DownloadResource error",e); + logger.error("DownloadResource error", e); Assert.assertTrue(false); } } @Test - public void testUnauthorizedFile(){ + public void testUnauthorizedFile() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.unauthorizedFile(user,1); + Map result = resourcesService.unauthorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(resourcesMapper.queryResourceExceptUserId(1)).thenReturn(getResourceList()); - result = resourcesService.unauthorizedFile(user,1); + result = resourcesService.unauthorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List resources = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); } @Test - public void testUnauthorizedUDFFunction(){ + public void testUnauthorizedUDFFunction() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.unauthorizedUDFFunction(user,1); + Map result = resourcesService.unauthorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(udfFunctionMapper.queryUdfFuncExceptUserId(1)).thenReturn(getUdfFuncList()); - result = resourcesService.unauthorizedUDFFunction(user,1); + result = resourcesService.unauthorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List udfFuncs = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); } - @Test - public void testAuthorizedUDFFunction(){ + public void testAuthorizedUDFFunction() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.authorizedUDFFunction(user,1); + Map result = resourcesService.authorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(udfFunctionMapper.queryAuthedUdfFunc(1)).thenReturn(getUdfFuncList()); - result = resourcesService.authorizedUDFFunction(user,1); + result = resourcesService.authorizedUDFFunction(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List udfFuncs = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); } @Test - public void testAuthorizedFile(){ + public void testAuthorizedFile() { User user = getUser(); //USER_NO_OPERATION_PERM - Map result = resourcesService.authorizedFile(user,1); + Map result = resourcesService.authorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList()); - result = resourcesService.authorizedFile(user,1); + result = resourcesService.authorizedFile(user, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List resources = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(resources)); } + @Test + public void testCatFile() { + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + + //SUCCESS + try { + Mockito.when(hadoopUtils.exists(null)).thenReturn(true); + Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent()); - private List getResourceList(){ + List list = hadoopUtils.catFile(null, 1, 10); + Assert.assertNotNull(list); - List resources = new ArrayList<>(); + } catch (IOException e) { + logger.error("hadoop error", e); + } + } + + private List getResourceList() { + + List resources = new ArrayList<>(); resources.add(getResource()); return resources; } - private Tenant getTenant() { Tenant tenant = new Tenant(); tenant.setTenantCode("123"); return tenant; } - private Resource getResource(){ + private Resource getResource() { Resource resource = new Resource(); resource.setPid(-1); @@ -637,7 +650,7 @@ public class ResourcesServiceTest { return resource; } - private Resource getUdfResource(){ + private Resource getUdfResource() { Resource resource = new Resource(); resource.setUserId(1); @@ -648,27 +661,28 @@ public class ResourcesServiceTest { return resource; } - private UdfFunc getUdfFunc(){ + private UdfFunc getUdfFunc() { UdfFunc udfFunc = new UdfFunc(); udfFunc.setId(1); return udfFunc; } - private List getUdfFuncList(){ + private List getUdfFuncList() { List udfFuncs = new ArrayList<>(); udfFuncs.add(getUdfFunc()); return udfFuncs; } - private User getUser(){ + private User getUser() { User user = new User(); user.setId(1); user.setTenantId(1); return user; } - private List getContent(){ + + private List getContent() { List contentList = new ArrayList<>(); contentList.add("test"); return contentList; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index e3e33566e9..f0e3ec8214 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -30,13 +30,14 @@ import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.charset.UnsupportedCharsetException; import java.util.Optional; @@ -217,7 +218,7 @@ public class FileUtils { return false; } bufferedReader = new BufferedReader(new StringReader(content)); - bufferedWriter = new BufferedWriter(new FileWriter(distFile)); + bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(distFile), StandardCharsets.UTF_8)); char[] buf = new char[1024]; int len; while ((len = bufferedReader.read(buf)) != -1) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index fe9ac932c1..a0c2bf4e58 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -39,6 +39,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.PrivilegedExceptionAction; import java.util.Collections; @@ -263,7 +264,7 @@ public class HadoopUtils implements Closeable { } try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) { - BufferedReader br = new BufferedReader(new InputStreamReader(in)); + BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); Stream stream = br.lines().skip(skipLineNums).limit(limit); return stream.collect(Collectors.toList()); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index b2a255b2e2..f87628c950 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -14,16 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; + import org.apache.dolphinscheduler.common.Constants; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; @RunWith(PowerMockRunner.class) @PrepareForTest(DateUtils.class) @@ -81,4 +88,15 @@ public class FileUtilsTest { } } + @Test + public void testWriteContent2File() throws FileNotFoundException { + // file exists, fmt is invalid + String filePath = "test/testFile.txt"; + String content = "正正正faffdasfasdfas"; + FileUtils.writeContent2File(content, filePath); + + String fileContent = FileUtils.readFile2Str(new FileInputStream(new File(filePath))); + Assert.assertEquals(content, fileContent); + } + }