[Bug][Common] read file garbled (#3479)

* fix bug : Random code problem

Co-authored-by: zhanglong <zhanglong@ysstech.com>

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java

@@ -14,12 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.api.service;
 
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import org.apache.dolphinscheduler.api.enums.Status;
-import org.apache.dolphinscheduler.api.exceptions.ServiceException;
 import org.apache.dolphinscheduler.api.utils.PageInfo;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
@@ -33,7 +31,18 @@ import org.apache.dolphinscheduler.dao.entity.Resource;
 import org.apache.dolphinscheduler.dao.entity.Tenant;
 import org.apache.dolphinscheduler.dao.entity.UdfFunc;
 import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.*;
+import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
+import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
+import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
+import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -49,10 +58,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.mock.web.MockMultipartFile;
 
-import java.io.IOException;
+import com.baomidou.mybatisplus.core.metadata.IPage;
-import java.util.ArrayList;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import java.util.List;
-import java.util.Map;
 
 @RunWith(PowerMockRunner.class)
 @PowerMockIgnore({"sun.security.*", "javax.net.*"})
@@ -104,14 +111,14 @@ public class ResourcesServiceTest {
         Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
 
         //RESOURCE_FILE_IS_EMPTY
-        MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf",new String().getBytes());
+        MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
         PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
         result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/");
         logger.info(result.toString());
         Assert.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
 
         //RESOURCE_SUFFIX_FORBID_CHANGE
-        mockMultipartFile = new MockMultipartFile("test.pdf","test.pdf","pdf",new String("test").getBytes());
+        mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
         PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf");
         PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
         result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/");
@@ -119,13 +126,12 @@ public class ResourcesServiceTest {
         Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
 
         //UDF_RESOURCE_SUFFIX_NOT_JAR
-        mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf","ResourcesServiceTest.pdf","pdf",new String("test").getBytes());
+        mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes());
         PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf");
         result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest", ResourceType.UDF, mockMultipartFile, -1, "/");
         logger.info(result.toString());
         Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
 
-
     }
 
     @Test
@@ -225,7 +231,6 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
 
-
         //SUCCESS
         Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
         PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(), Mockito.any())).thenReturn("test");
@@ -340,7 +345,6 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
 
-
         //RESOURCE_FILE_EXIST
         user.setTenantId(1);
         try {
@@ -377,7 +381,6 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
 
-
         //RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
         PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("class");
         PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
@@ -392,14 +395,12 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
 
-
         //TENANT_NOT_EXIST
         Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
         result = resourcesService.readResource(1, 1, 10);
         logger.info(result.toString());
         Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
 
-
         //RESOURCE_FILE_NOT_EXIST
         Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
         try {
@@ -422,7 +423,6 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
 
-
     }
 
     @Test
@@ -461,7 +461,6 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
 
-
     }
 
     @Test
@@ -497,7 +496,6 @@ public class ResourcesServiceTest {
         logger.info(result.toString());
         Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
 
-
         //TENANT_NOT_EXIST
         Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
         result = resourcesService.updateResourceContent(1, "content");
@@ -574,7 +572,6 @@ public class ResourcesServiceTest {
         Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncs));
     }
 
-
     @Test
     public void testAuthorizedUDFFunction() {
         User user = getUser();
@@ -610,6 +607,23 @@ public class ResourcesServiceTest {
         Assert.assertTrue(CollectionUtils.isNotEmpty(resources));
     }
 
+    @Test
+    public void testCatFile() {
+
+        PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
+
+        //SUCCESS
+        try {
+            Mockito.when(hadoopUtils.exists(null)).thenReturn(true);
+            Mockito.when(hadoopUtils.catFile(null, 1, 10)).thenReturn(getContent());
+
+            List<String> list = hadoopUtils.catFile(null, 1, 10);
+            Assert.assertNotNull(list);
+
+        } catch (IOException e) {
+            logger.error("hadoop error", e);
+        }
+    }
 
     private List<Resource> getResourceList() {
 
@@ -618,7 +632,6 @@ public class ResourcesServiceTest {
         return resources;
     }
 
-
     private Tenant getTenant() {
         Tenant tenant = new Tenant();
         tenant.setTenantCode("123");
@@ -668,6 +681,7 @@ public class ResourcesServiceTest {
         user.setTenantId(1);
         return user;
     }
+
     private List<String> getContent() {
         List<String> contentList = new ArrayList<>();
         contentList.add("test");

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@@ -30,13 +30,14 @@ import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.StringReader;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.charset.UnsupportedCharsetException;
 import java.util.Optional;
 
@@ -217,7 +218,7 @@ public class FileUtils {
                 return false;
             }
             bufferedReader = new BufferedReader(new StringReader(content));
-            bufferedWriter = new BufferedWriter(new FileWriter(distFile));
+            bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(distFile), StandardCharsets.UTF_8));
             char[] buf = new char[1024];
             int len;
             while ((len = bufferedReader.read(buf)) != -1) {

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java

@@ -39,6 +39,7 @@ import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
@@ -263,7 +264,7 @@ public class HadoopUtils implements Closeable {
         }
 
         try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) {
-            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
             Stream<String> stream = br.lines().skip(skipLineNums).limit(limit);
             return stream.collect(Collectors.toList());
         }

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@@ -14,16 +14,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.common.utils;
 
+import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
+
 import org.apache.dolphinscheduler.common.Constants;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
-import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(DateUtils.class)
@@ -81,4 +88,15 @@ public class FileUtilsTest {
         }
     }
 
+    @Test
+    public void testWriteContent2File() throws FileNotFoundException {
+        // file exists, fmt is invalid
+        String filePath = "test/testFile.txt";
+        String content = "正正正faffdasfasdfas";
+        FileUtils.writeContent2File(content, filePath);
+
+        String  fileContent = FileUtils.readFile2Str(new FileInputStream(new File(filePath)));
+        Assert.assertEquals(content, fileContent);
+    }
+
 }