From 63f835715f8ca8bff79c0e7177ebfa5917ebb3bd Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Wed, 18 May 2022 18:40:17 +0800 Subject: [PATCH] [fix] Enhance name pre checker in resource center (#10094) * [fix] Enhance name pre checker in resource center Add file name and directory checker to avoid directory traversal * add some missing change and change docs * change var name in directoryTraversal * Fix ci --- docs/docs/en/guide/resource/file-manage.md | 31 ++++++++++++------- docs/docs/zh/guide/resource/file-manage.md | 31 +++++++++++-------- .../dolphinscheduler/api/enums/Status.java | 2 +- .../service/impl/ResourcesServiceImpl.java | 21 +++++++++++-- .../common/utils/FileUtils.java | 21 +++++++++++++ .../common/utils/FileUtilsTest.java | 30 ++++++++++++++++++ 6 files changed, 108 insertions(+), 28 deletions(-) diff --git a/docs/docs/en/guide/resource/file-manage.md b/docs/docs/en/guide/resource/file-manage.md index ae3e8d2ec4..87b965801b 100644 --- a/docs/docs/en/guide/resource/file-manage.md +++ b/docs/docs/en/guide/resource/file-manage.md @@ -4,40 +4,47 @@ When third party jars are used in the scheduling process or user defined scripts ![file-manage](/img/new_ui/dev/resource/file-manage.png) -- Create a file - > The file format supports the following types: txt, log, sh, conf, cfg, py, java, sql, xml, hql, properties. +## Basic Operator + +### Create a File + +The file format supports the following types: txt, log, sh, conf, cfg, py, java, sql, xml, hql, properties. ![create-file](/img/new_ui/dev/resource/create-file.png) -- upload files +### Upload Files -> Upload file: Click the "Upload File" button to upload, drag the file to the upload area, the file name will be automatically completed with the uploaded file name. +Click the "Upload File" button to upload, drag the file to the upload area, the file name will be automatically completed with the uploaded file name. ![upload-file](/img/new_ui/dev/resource/upload-file.png) -- File View +### View File Content -> For the files that can be viewed, click the file name to view the file details. + For the files that can be viewed, click the file name to view the file details. ![file_detail](/img/tasks/demo/file_detail.png) -- Download file +### Download file > Click the "Download" button in the file list to download the file or click the "Download" button in the upper right corner of the file details to download the file. -- File rename +### Rename File ![rename-file](/img/new_ui/dev/resource/rename-file.png) -- delete - > File list -> Click the "Delete" button to delete the specified file. +### Delete File -- Re-upload file +File list -> Click the "Delete" button to delete the specified file. - > Re-upload file: Click the "Re-upload File" button to upload a new file to replace the old file, drag the file to the re-upload area, the file name will be automatically completed with the new uploaded file name. +### Re-upload file + +Click the "Re-upload File" button to upload a new file to replace the old file, drag the file to the re-upload area, the file name will be automatically completed with the new uploaded file name. ![reuplod_file](/img/reupload_file_en.png) +> Note: File name or source name of your local file can not contain specific characters like `.` or `/` when you trying to +> upload, create or rename file in resource center. + ## Example The example uses a simple shell script to demonstrate the use of resource center files in workflow definitions. The same is true for tasks such as MR and Spark, which require jar packages. diff --git a/docs/docs/zh/guide/resource/file-manage.md b/docs/docs/zh/guide/resource/file-manage.md index 77d9509548..a0d2f0ab56 100644 --- a/docs/docs/zh/guide/resource/file-manage.md +++ b/docs/docs/zh/guide/resource/file-manage.md @@ -2,43 +2,48 @@ 当在调度过程中需要使用到第三方的 jar 或者用户需要自定义脚本的情况，可以通过在该页面完成相关操作。可创建的文件类型包括：`txt/log/sh/conf/py/java` 等。并且可以对文件进行编辑、重命名、下载和删除等操作。 +## 基础操作 + ![file-manage](/img/new_ui/dev/resource/file-manage.png) -* 创建文件 +### 创建文件 - > 文件格式支持以下几种类型：txt、log、sh、conf、cfg、py、java、sql、xml、hql、properties +文件格式支持以下几种类型：txt、log、sh、conf、cfg、py、java、sql、xml、hql、properties ![create-file](/img/new_ui/dev/resource/create-file.png) -* 上传文件 - > 上传文件：点击"上传文件"按钮进行上传，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全 +### 上传文件 + +上传文件：点击"上传文件"按钮进行上传，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全 ![upload-file](/img/new_ui/dev/resource/upload-file.png) -* 文件查看 +### 文件查看 - > 对可查看的文件类型，点击文件名称，可查看文件详情 +对可查看的文件类型，点击文件名称，可查看文件详情 ![file_detail](/img/tasks/demo/file_detail.png) -* 下载文件 +### 下载文件 - > 点击文件列表的"下载"按钮下载文件或者在文件详情中点击右上角"下载"按钮下载文件 +点击文件列表的"下载"按钮下载文件或者在文件详情中点击右上角"下载"按钮下载文件 -* 文件重命名 +### 文件重命名 ![rename-file](/img/new_ui/dev/resource/rename-file.png) -* 删除 +### 删除文件 -> 文件列表->点击"删除"按钮，删除指定文件 +文件列表->点击"删除"按钮，删除指定文件 -* 重新上传文件 +### 重新上传文件 - > 点击文件列表中的”重新上传文件“按钮进行重新上传文件，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全 +点击文件列表中的”重新上传文件“按钮进行重新上传文件，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全 ![reuplod_file](/img/reupload_file_en.png) +> 注意：上传、创建、重命名文件时，文件名和源文件名（上传时）均不能带有 `.` 以及 `/` 特殊符号。 + ## 任务样例 该样例主要通过一个简单的 shell 脚本，来演示如何在工作流定义中使用资源中心的文件。像 MR、Spark 等任务需要用到 jar 包，也是同理。 diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index e0a38c1e5f..f5313f053b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -396,7 +396,7 @@ public enum Status { K8S_CLIENT_OPS_ERROR(1300006, "k8s error with exception {0}", "k8s操作报错[{0}]"), VERIFY_K8S_NAMESPACE_ERROR(1300007, "verify k8s and namespace error", "验证k8s命名空间信息错误"), DELETE_K8S_NAMESPACE_BY_ID_ERROR(1300008, "delete k8s namespace by id error", "删除命名空间错误"), - VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed", "文件命名校验失败"), + VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed", "文件命名校验失败"), STORE_OPERATE_CREATE_ERROR(1300010, "create the resource failed", "存储操作失败"), GRANT_K8S_NAMESPACE_ERROR(1300011, "grant namespace error", "授权资源错误"), QUERY_UNAUTHORIZED_NAMESPACE_ERROR(1300012, "query unauthorized namespace error", "查询未授权命名空间错误"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 200aeb1c65..1c5804cd81 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -129,8 +129,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } - if (name.endsWith(FOLDER_SEPARATOR)) { - result.setCode(Status.VERIFY_PARAMETER_NAME_FAILED.getCode()); + if (FileUtils.directoryTraversal(name)) { + putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); return result; } String fullName = getFullName(currentDir, name); @@ -517,6 +517,19 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe private Result verifyFile(String name, ResourceType type, MultipartFile file) { Result result = new Result<>(); putMsg(result, Status.SUCCESS); + + if (FileUtils.directoryTraversal(name)) { + logger.error("file alias name {} verify failed", name); + putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); + return result; + } + + if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) { + logger.error("file original name {} verify failed", file.getOriginalFilename()); + putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); + return result; + } + if (file != null) { // file is empty if (file.isEmpty()) { @@ -973,6 +986,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } + if (FileUtils.directoryTraversal(fileName)) { + putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); + return result; + } //check file suffix String nameSuffix = fileSuffix.trim(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index e7817ffe80..bdcf62f76a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -204,4 +204,25 @@ public class FileUtils { } } + /** + * Check whether the given string type of path can be traversal or not, return true if path could + * traversal, and return false if it is not. + * + * @param filename String type of filename + * @return whether file path could be traversal or not + */ + public static boolean directoryTraversal(String filename){ + if (filename.contains(FOLDER_SEPARATOR)) { + return true; + } + File file = new File(filename); + try { + File canonical = file.getCanonicalFile(); + File absolute = file.getAbsoluteFile(); + return !canonical.equals(absolute); + } catch (IOException e) { + return true; + } + } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index d023b8651f..fdcaccd697 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -88,4 +88,34 @@ public class FileUtilsTest { Assert.assertEquals(content, fileContent); } + @Test + public void testDirectoryTraversal() { + // test case which do not directory traversal + String path; + path = "abc.txt"; + Assert.assertFalse(FileUtils.directoryTraversal(path)); + + path = "abc...txt"; + Assert.assertFalse(FileUtils.directoryTraversal(path)); + + path = "..abc.txt"; + Assert.assertFalse(FileUtils.directoryTraversal(path)); + + // test case which will directory traversal + path = "../abc.txt"; + Assert.assertTrue(FileUtils.directoryTraversal(path)); + + path = "../../abc.txt"; + Assert.assertTrue(FileUtils.directoryTraversal(path)); + + path = "abc../def.txt"; + Assert.assertTrue(FileUtils.directoryTraversal(path)); + + path = "abc./def.txt"; + Assert.assertTrue(FileUtils.directoryTraversal(path)); + + path = "abc/def...txt"; + Assert.assertTrue(FileUtils.directoryTraversal(path)); + } + }