Pull request #89: Release/10.0

Merge in PG/plugin-repository-s3 from release/10.0 to persist/10.0

* commit '8533a78ee98a993ebd9030731c2a2762f55dbfc1':
  REPORT-87924 fix:增加输入限制
  REPORT-87924 feat:适配S3插件前台改动
  REPORT-88094 fix: change-notes
  REPORT-88094 fix: 文件数量较多时，listEntry结果不全
  无jira任务 perf: 优化写文件性能

plugin.xml

@@ -5,12 +5,13 @@
     <main-package>com.fanruan.fs</main-package>
     <active>yes</active>
     <hidden>no</hidden>
-    <version>1.3.5</version>
+    <version>1.3.6</version>
     <env-version>10.0~10.0</env-version>
     <jartime>2021-03-11</jartime>
     <vendor>richie</vendor>
     <description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
     <change-notes><![CDATA[
+        [2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/>
         [2022-09-22]第三方组件升级。 <br/>
         [2022-07-25]修复http无法连接，而https和缺省的时候正常<br/>
         [2022-06-30]新增enablePathStyleAccess、signerOverride后台配置<br/>

src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@@ -87,6 +87,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
         ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                 .withPrefix(dir).withDelimiter(DELIMITER);
         ObjectListing objectListing = s3.listObjects(listObjectsRequest);
+        collectFileEntry(result, objectListing);
+        while (objectListing.isTruncated()) {
+            objectListing = s3.listNextBatchOfObjects(objectListing);
+            collectFileEntry(result, objectListing);
+        }
+        return result.toArray(new FineFileEntry[0]);
+    }
+
+    private void collectFileEntry(List<FineFileEntry> result, ObjectListing objectListing) {
         for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
             String key = summary.getKey();
             if (!key.endsWith(DELIMITER)) {
@@ -98,8 +107,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
             entry.setDirectory(true);
             result.add(entry);
         }
-
-        return result.toArray(new FineFileEntry[0]);
     }
 
     @Override
@@ -119,17 +126,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     @Override
     public void write(String path, byte[] data) {
-
-        List<String> paths = getAllNecessaryPath(path);
-        for (int i = paths.size() - 1; i > 0; i--) {
-            String parent = paths.get(i);
-            PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
-            try {
-                s3.putObject(req);
-            } catch (Exception e) {
-                LogKit.error("[S3] Failed to create parent path {}", parent);
-            }
-        }
         ObjectMetadata metadata;
         try {
             metadata = s3.getObjectMetadata(bucket, path);
@@ -150,16 +146,16 @@ public class S3ResourceRepository extends BaseResourceRepository {
     @Override
     public boolean createFile(String path) {
 
-        List<String> paths = getAllNecessaryPath(path);
+        String parent = path.substring(0, path.lastIndexOf(DELIMITER) + 1);
-        for (int i = paths.size() - 1; i >= 0; i--) {
+        if (!dirExist(parent)) {
-            String parent = paths.get(i);
+            createDirectory(parent);
-            PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
+        }
-            try {
+        PutObjectRequest req = new PutObjectRequest(bucket, path, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
-                s3.putObject(req);
+        try {
-            } catch (Exception e) {
+            s3.putObject(req);
-                LogKit.error("[S3] Failed to create parent path {}", parent);
+        } catch (Exception e) {
-                return false;
+            LogKit.error("[S3] Failed to create file path {}", path);
-            }
+            return false;
         }
         return true;
     }
@@ -170,17 +166,23 @@ public class S3ResourceRepository extends BaseResourceRepository {
         if (!path.endsWith(DELIMITER)) {
             path += DELIMITER;
         }
-        List<String> paths = getAllNecessaryPath(path);
+        String temp = path.substring(0, path.length() - 1);
-        for (int i = paths.size() - 1; i >= 0; i--) {
+        String parent = temp.substring(0, temp.lastIndexOf(DELIMITER) + 1);
-            String parent = paths.get(i);
+        if (StringUtils.isEmpty(parent)) {
-            PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
+            return true;
-            try {
+        }
-                s3.putObject(req);
+        if (!dirExist(parent)) {
-            } catch (Exception e) {
+            if (!createDirectory(parent)) {
-                LogKit.error("[S3] Failed to create parent path {}", parent);
                 return false;
             }
         }
+        PutObjectRequest req = new PutObjectRequest(bucket, path, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
+        try {
+            s3.putObject(req);
+        } catch (Exception e) {
+            LogKit.error("[S3] Failed to create parent path {}", path);
+            return false;
+        }
         return true;
     }
 
@@ -214,13 +216,49 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     @Override
     public boolean exist(String path) {
+        return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path);
+    }
+
+    private boolean fileExist(String path) {
         try {
-            return s3.doesObjectExist(bucket, path) || s3.doesObjectExist(bucket, path + DELIMITER);
+            return s3.doesObjectExist(bucket, path);
         } catch (Exception e) {
             return false;
         }
     }
 
+    private boolean dirExist(String path) {
+        if (StringUtils.equals(path, DELIMITER)) {
+            return true;
+        }
+        if (!path.endsWith(DELIMITER)) {
+            path += DELIMITER;
+        }
+        return fileExist(path);
+    }
+
+    /**
+     * 如果存在文件创建了，但是其父目录没有创建的场景，为其递归创建对象，返回创建结果
+     */
+    private boolean isParentPathAbsent(String path) {
+        if (path.startsWith(DELIMITER)) {
+            path = path.substring(1);
+        }
+        if (path.endsWith(DELIMITER)) {
+            path = path.substring(0, path.length() - 1);
+        }
+        ObjectListing objectListing = s3.listObjects(
+                new ListObjectsRequest()
+                        .withBucketName(bucket)
+                        .withPrefix(path));
+        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
+            if (summary.getKey().startsWith(path + DELIMITER)) {
+                return createDirectory(path);
+            }
+        }
+        return false;
+    }
+
     @Override
     public String[] list(String dir, final Filter<String> filter) {
         List<String> result = new ArrayList<>();

src/main/resources/com/fanruan/fs/s3/repository/locale/s3.properties

@@ -4,4 +4,6 @@ Plugin-S3_Region=Region
 Plugin-S3_Access_Key_Id=AccessKeyId
 Plugin-S3_Access_Key_Secret=AccessKeySecret
 Plugin-S3_Bucket=Bucket
-Dec-Error_Start_With_Slash_Or_End_Without_Slash=The path cannot start with "/", but must end with "/"
+Dec-Error_Start_With_Slash_Or_End_Without_Slash=The path cannot start with "/", but must end with "/"
+Plugin-S3_Other_Config=
+Plugin-S3_EnablePathStyleAccess_Error_Tip=

src/main/resources/com/fanruan/fs/s3/repository/locale/s3_en_US.properties

@@ -4,4 +4,6 @@ Plugin-S3_Access_Key_Secret=AccessKeySecret
 Plugin-S3_Bucket=Bucket
 Plugin-S3_End_Point=Endpoint
 Plugin-S3_Input=Please input
-Plugin-S3_Region=Region
+Plugin-S3_Region=Region
+Plugin-S3_Other_Config=
+Plugin-S3_EnablePathStyleAccess_Error_Tip=

src/main/resources/com/fanruan/fs/s3/repository/locale/s3_ja_JP.properties

@@ -4,4 +4,6 @@ Plugin-S3_Access_Key_Secret=AccessKeySecret
 Plugin-S3_Bucket=Bucket
 Plugin-S3_End_Point=Endpoint
 Plugin-S3_Input=\u5165\u529B\u3057\u3066\u304F\u3060\u3055\u3044\u3002
-Plugin-S3_Region=Region
+Plugin-S3_Region=Region
+Plugin-S3_Other_Config=
+Plugin-S3_EnablePathStyleAccess_Error_Tip=

src/main/resources/com/fanruan/fs/s3/repository/locale/s3_ko_KR.properties

@@ -4,4 +4,6 @@ Plugin-S3_Access_Key_Secret=
 Plugin-S3_Bucket=
 Plugin-S3_End_Point=
 Plugin-S3_Input=
-Plugin-S3_Region=
+Plugin-S3_Region=
+Plugin-S3_Other_Config=
+Plugin-S3_EnablePathStyleAccess_Error_Tip=

src/main/resources/com/fanruan/fs/s3/repository/locale/s3_zh_CN.properties

@@ -4,4 +4,6 @@ Plugin-S3_Access_Key_Secret=AccessKeySecret
 Plugin-S3_Bucket=Bucket
 Plugin-S3_End_Point=Endpoint
 Plugin-S3_Input=\u8BF7\u8F93\u5165
-Plugin-S3_Region=Region
+Plugin-S3_Region=Region
+Plugin-S3_Other_Config=\u66f4\u591a\u8bbe\u7f6e
+Plugin-S3_EnablePathStyleAccess_Error_Tip=\u8bf7\u8f93\u5165true\u6216false

src/main/resources/com/fanruan/fs/s3/repository/locale/s3_zh_TW.properties

@@ -4,4 +4,6 @@ Plugin-S3_Access_Key_Secret=AccessKeySecret
 Plugin-S3_Bucket=Bucket
 Plugin-S3_End_Point=Endpoint
 Plugin-S3_Input=\u8ACB\u8F38\u5165
-Plugin-S3_Region=Region
+Plugin-S3_Region=Region
+Plugin-S3_Other_Config=
+Plugin-S3_EnablePathStyleAccess_Error_Tip=

src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js

@@ -137,11 +137,77 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
                         ref: function (_ref) {
                             self.filePathRow = _ref;
                         },
-                    }],
+                    }, {
+                        type: "bi.vertical_adapt",
+                        items: [{
+                            type: "bi.icon_change_button",
+                            iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font",
+                            ref: (_ref) => {
+                                this.OtherConfigButton = _ref;
+                            },
+                            handler: () => {
+                                this.store.setIsOpen(!this.model.isOpen);
+                                this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
+                            }
+                        }, {
+                            type: "bi.text_button",
+                            text: BI.i18nText('Plugin-S3_Other_Config'),
+                            handler: () => {
+                                this.store.setIsOpen(!this.model.isOpen);
+                                this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
+                            }
+                        }]
+                    }, {
+                        type: 'bi.vertical',
+                        invisible: () => !this.model.isOpen,
+                        items: [{
+                            type: "dec.label.editor.item",
+                            textWidth: LABEL_WIDTH,
+                            editorWidth: EDITOR_WIDTH,
+                            watermark: BI.i18nText("Plugin-S3_Input"),
+                            text: "PathStyleAccess",
+                            value: this.model.enablePathStyleAccess,
+                            ref: function (_ref) {
+                                self.enablePathStyleAccessRow = _ref;
+                            },
+                            el: {
+                                disabled: !o.editable,
+                            },
+                            listeners: [{
+                                eventName: BI.Editor.EVENT_CHANGE,
+                                action: function () {
+                                    self.store.setEnablePathStyleAccess(this.getValue());
+                                }
+                            }]
+                        }, {
+                            type: "dec.label.editor.item",
+                            textWidth: LABEL_WIDTH,
+                            editorWidth: EDITOR_WIDTH,
+                            watermark: BI.i18nText("Plugin-S3_Input"),
+                            text: "SignerOverride",
+                            value: this.model.signerOverride,
+                            el: {
+                                disabled: !o.editable,
+                            },
+                            tgap: 15,
+                            listeners: [{
+                                eventName: BI.Editor.EVENT_CHANGE,
+                                action: function () {
+                                    self.store.setSignerOverride(this.getValue());
+                                }
+                            }]
+                        }],
+                    }
+                ]
             };
         },
 
         getValue: function () {
+            var enablePathStyleAccess = false;
+            if (this.model.enablePathStyleAccess === 'true') {
+                enablePathStyleAccess = true;
+            }
+
             return {
                 endPoint: this.model.endPoint,
                 region: this.model.region,
@@ -149,6 +215,8 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
                 password: this.passwordRow.getCipher(),
                 bucket: this.model.bucket,
                 workRoot: this.filePathRow.getValue(),
+                enablePathStyleAccess,
+                signerOverride: this.model.signerOverride,
             };
         },
 
@@ -167,6 +235,10 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
                 this.filePathRow.showError(BI.i18nText("Dec-Error_Null"));
                 valid = false;
             }
+            if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) {
+                this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip"));
+                valid = false;
+            }
 
             return valid;
         },
@@ -189,6 +261,9 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
                 password: val.password,
                 bucket: val.bucket,
                 workRoot: val.workRoot,
+                isOpen: false,
+                enablePathStyleAccess: 'false',
+                signerOverride: "",
             };
         },
 
@@ -218,6 +293,18 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
             setBucket: function (v) {
                 this.model.bucket = v;
             },
+
+            setEnablePathStyleAccess: function (v) {
+                this.model.enablePathStyleAccess = v;
+            },
+
+            setSignerOverride: function (v) {
+                this.model.signerOverride = v;
+            },
+
+            setIsOpen: function (v) {
+                this.model.isOpen = v;
+            },
         },
     });
     BI.model("dec.model.intelligence.cluster.file.s3", Model);