Fix createFile with permission will not work (#15556)

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java

@@ -25,8 +25,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VI
 import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8;
 import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
 
-import org.apache.dolphinscheduler.common.exception.FileOperateException;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SystemUtils;
 
@@ -38,9 +36,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
-import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.Set;
@@ -48,11 +44,10 @@ import java.util.zip.CRC32;
 import java.util.zip.CheckedInputStream;
 
 import lombok.NonNull;
+import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
-/**
+@UtilityClass
- * file utils
- */
 @Slf4j
 public class FileUtils {
 
@@ -62,14 +57,7 @@ public class FileUtils {
 
     public static final String KUBE_CONFIG_FILE = "config";
 
-    private static final String RWXR_XR_X = "rwxr-xr-x";
+    private static final Set<PosixFilePermission> PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x");
-
-    private static final FileAttribute<Set<PosixFilePermission>> PERMISSION_755 =
-            PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(RWXR_XR_X));
-
-    private FileUtils() {
-        throw new UnsupportedOperationException("Construct FileUtils");
-    }
 
     /**
      * get download file absolute path and name
@@ -161,34 +149,6 @@ public class FileUtils {
         return PropertyUtils.getString(RESOURCE_VIEW_SUFFIXES, RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE);
     }
 
-    /**
-     * create directory if absent
-     *
-     * @param execLocalPath execute local path
-     * @throws IOException errors
-     */
-    public static void createWorkDirIfAbsent(String execLocalPath) throws IOException {
-        // if work dir exists, first delete
-        File execLocalPathFile = new File(execLocalPath);
-
-        if (execLocalPathFile.exists()) {
-            try {
-                org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
-            } catch (Exception ex) {
-                if (ex instanceof NoSuchFileException || ex.getCause() instanceof NoSuchFileException) {
-                    // this file is already be deleted.
-                } else {
-                    throw ex;
-                }
-            }
-        }
-
-        // create work dir
-        org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
-        String mkdirLog = "create dir success " + execLocalPath;
-        log.info(mkdirLog);
-    }
-
     /**
      * write content to file ,if parent path not exists, it will do one's utmost to mkdir
      *
@@ -231,25 +191,6 @@ public class FileUtils {
         org.apache.commons.io.FileUtils.deleteQuietly(new File(filename));
     }
 
-    /**
-     * Gets all the parent subdirectories of the parentDir directory
-     *
-     * @param parentDir parent dir
-     * @return all dirs
-     */
-    public static File[] getAllDir(String parentDir) {
-        if (parentDir == null || "".equals(parentDir)) {
-            throw new RuntimeException("parentDir can not be empty");
-        }
-
-        File file = new File(parentDir);
-        if (!file.exists() || !file.isDirectory()) {
-            throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
-        }
-
-        return file.listFiles(File::isDirectory);
-    }
-
     /**
      * Get Content
      *
@@ -325,59 +266,47 @@ public class FileUtils {
         return crcString;
     }
 
-    public static void setFileOwner(Path filePath, String fileOwner) throws FileOperateException {
+    public static void createFileWith755(@NonNull Path path) throws IOException {
-        try {
+        if (SystemUtils.IS_OS_WINDOWS) {
-            // We use linux command to set the file owner, since jdk api will not use sudo.
+            Files.createFile(path);
-            String command = String.format("sudo chown %s %s", fileOwner, filePath.toString());
+        } else {
-            Runtime.getRuntime().exec(command);
+            Files.createFile(path);
-            Process process = Runtime.getRuntime().exec(command);
+            Files.setPosixFilePermissions(path, PERMISSION_755);
-            int exitCode = process.waitFor();
+        }
-            if (0 != exitCode) {
-                throw new FileOperateException(
-                        "Set file: " + filePath + " to owner: " + fileOwner + " failed, existCode(" + exitCode + ")");
     }
-        } catch (FileOperateException ex) {
-            throw ex;
-        } catch (Exception ex) {
-            throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
 
+    public static void createDirectoryWith755(@NonNull Path path) throws IOException {
+        if (path.toFile().exists()) {
+            return;
         }
+        if (OSUtils.isWindows()) {
+            Files.createDirectories(path);
+        } else {
+            Path parent = path.getParent();
+            if (parent != null && !parent.toFile().exists()) {
+                createDirectoryWith755(parent);
             }
 
-    public static void setDirectoryOwner(Path filePath, String fileOwner) throws FileOperateException {
+            Files.createDirectory(path);
-        try {
+            Files.setPosixFilePermissions(path, PERMISSION_755);
-            // We use linux command to set the file owner, since jdk api will not use sudo.
-            String command = String.format("sudo chown -R %s %s", fileOwner, filePath.toString());
-            Runtime.getRuntime().exec(command);
-            Process process = Runtime.getRuntime().exec(command);
-            int exitCode = process.waitFor();
-            if (0 != exitCode) {
-                throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner
-                        + " failed, existCode(" + exitCode + ")");
-            }
-        } catch (FileOperateException ex) {
-            throw ex;
-        } catch (Exception ex) {
-            throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
 
         }
     }
 
-    public static void createDirectoryIfNotPresent(Path path) throws IOException {
+    public static void setFileTo755(File file) throws IOException {
-        if (Files.exists(path)) {
+        if (OSUtils.isWindows()) {
             return;
         }
-        Files.createDirectories(path);
+        if (file.isFile()) {
+            Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
+            return;
+        }
+        Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
+        File[] files = file.listFiles();
+        if (files != null) {
+            for (File f : files) {
+                setFileTo755(f);
             }
-
-    /**
-     * Create a file with '755'.
-     */
-    public static void createFileWith755(@NonNull Path path) throws IOException {
-        if (SystemUtils.IS_OS_WINDOWS) {
-            Files.createFile(path);
-        } else {
-            Files.createFile(path, PERMISSION_755);
         }
     }
 

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@@ -19,8 +19,13 @@ package org.apache.dolphinscheduler.common.utils;
 
 import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
@@ -55,12 +60,23 @@ public class FileUtilsTest {
     }
 
     @Test
-    public void testCreateWorkDirIfAbsent() {
+    public void createDirectoryWith755() throws IOException {
+        Path path = Paths.get("/tmp/createWorkDirAndUserIfAbsent");
         try {
-            FileUtils.createWorkDirIfAbsent("/tmp/createWorkDirAndUserIfAbsent");
+            FileUtils.createDirectoryWith755(path);
-            Assertions.assertTrue(true);
+            File file = path.toFile();
+            Assertions.assertTrue(file.exists());
+            Assertions.assertTrue(file.isDirectory());
+            Assertions.assertTrue(file.canExecute());
+            Assertions.assertTrue(file.canRead());
+            Assertions.assertTrue(file.canWrite());
+
+            FileUtils.createDirectoryWith755(Paths.get("/"));
         } catch (Exception e) {
-            Assertions.fail();
+            e.printStackTrace();
+            Assertions.fail(e.getMessage());
+        } finally {
+            Files.deleteIfExists(path);
         }
     }
 

dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java

@@ -30,6 +30,7 @@ import org.apache.commons.lang3.SystemUtils;
 
 import java.io.File;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -176,7 +177,7 @@ public class ProcessUtils {
                                     taskExecutionContext.getProcessInstanceId(),
                                     taskExecutionContext.getTaskInstanceId()));
                 }
-                FileUtils.createWorkDirIfAbsent(taskExecutionContext.getExecutePath());
+                FileUtils.createDirectoryWith755(Paths.get(taskExecutionContext.getExecutePath()));
                 org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils.cancelApplication(taskExecutionContext);
                 return appIds;
             } else {

dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java

@@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -170,7 +171,7 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
 
         BlobClient blobClient = blobContainerClient.getBlobClient(srcFilePath);

dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java

@@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -143,7 +144,7 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
 
         Blob blob = gcsStorage.get(BlobId.of(bucketName, srcFilePath));

dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java

@@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -187,7 +188,7 @@ public class ObsStorageOperator implements Closeable, StorageOperate {
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
         ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath);
         try (

dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java

@@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
 import org.apache.dolphinscheduler.common.factory.OssClientFactory;
 import org.apache.dolphinscheduler.common.model.OssConnection;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -213,7 +214,7 @@ public class OssStorageOperator implements Closeable, StorageOperate {
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
         OSSObject ossObject = ossClient.getObject(bucketName, srcFilePath);
         try (

dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java

@@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
 
 import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -200,7 +201,7 @@ public class S3StorageOperator implements Closeable, StorageOperate {
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {
-            Files.createDirectories(dstFile.getParentFile().toPath());
+            FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
         }
         S3Object o = s3Client.getObject(bucketName, srcFilePath);
         try (

dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java

@@ -17,8 +17,6 @@
 
 package org.apache.dolphinscheduler.plugin.task.api.shell;
 
-import org.apache.dolphinscheduler.common.constants.TenantConstants;
-import org.apache.dolphinscheduler.common.exception.FileOperateException;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.task.api.utils.AbstractCommandExecutorConstants;
@@ -71,12 +69,8 @@ public abstract class BaseLinuxShellInterceptorBuilder<T extends BaseLinuxShellI
                 "****************************** Script Content *****************************************************************");
     }
 
-    protected List<String> generateBootstrapCommand() throws FileOperateException {
+    protected List<String> generateBootstrapCommand() {
         if (sudoEnable) {
-            if (!TenantConstants.BOOTSTRAPT_SYSTEM_USER.equals(runUser)) {
-                // Set the tenant owner as the working directory
-                FileUtils.setDirectoryOwner(Paths.get(shellDirectory), runUser);
-            }
             return bootstrapCommandInSudoMode();
         }
         return bootstrapCommandInNormalMode();

dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java

@@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
 import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR;
 import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAVA;
 
+import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -155,7 +156,7 @@ public class JavaTaskTest {
             try {
                 Path path = Paths.get(fileName);
                 if (!Files.exists(path)) {
-                    Files.createDirectories(path);
+                    FileUtils.createDirectoryWith755(path);
                 }
                 javaTask.createJavaSourceFileIfNotExists(sourceCode, fileName);
             } finally {

dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java

@@ -97,7 +97,7 @@ public class TaskExecutionContextUtils {
                 log.warn("The TaskInstance WorkingDirectory: {} is exist, will recreate again",
                         taskInstanceWorkingDirectory);
             }
-            Files.createDirectories(Paths.get(taskInstanceWorkingDirectory));
+            FileUtils.createDirectoryWith755(Paths.get(taskInstanceWorkingDirectory));
 
             taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory);
             taskExecutionContext.setAppInfoPath(FileUtils.getAppInfoPath(taskInstanceWorkingDirectory));
@@ -137,6 +137,7 @@ public class TaskExecutionContextUtils {
                     storageOperate.download(resourceAbsolutePathInStorage, resourceAbsolutePathInLocal, true);
                     log.debug("Download resource file {} under: {} successfully", resourceAbsolutePathInStorage,
                             resourceAbsolutePathInLocal);
+                    FileUtils.setFileTo755(file);
                     WorkerServerMetrics
                             .recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime);
                     WorkerServerMetrics

dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java

@@ -49,7 +49,7 @@ class TaskExecutionContextUtilsTest {
         try {
             // Test if the working directory is exist
             // will delete it and recreate
-            Files.createDirectories(Paths.get(taskWorkingDirectory));
+            FileUtils.createDirectoryWith755(Paths.get(taskWorkingDirectory));
             Files.createFile(Paths.get(taskWorkingDirectory, "text.txt"));
             Assertions.assertTrue(Files.exists(Paths.get(taskWorkingDirectory, "text.txt")));