Pull request #263: 11.0.17版本插件代码合并

Merge in PG/plugin-repository-s3 from release/11.0 to persist/11.0

* commit '706604a5afa4df364b6473f420060b7ffc82a100':
  无jira任务，s3插件默认参数读取
  REPORT-96529 fix: list接口数量限制;删除接口性能优化
  无jira任务，s3插件默认参数读取
  无jira任务，提升jartime和版本号
  REPORT-95454 bugfix：http配置根据endpoint进行设置
  REPORT-91781 fix: 三方组件升级
  REPORT-91281 fix:插件加密方法修改

.gitignore

@@ -7,4 +7,6 @@ target/
 build
 local.properties
 classes/
-transform-classes/
+transform-classes/
+node_modules/
+*.lock

front/bundle.js

@@ -0,0 +1,311 @@
+BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
+    items.push({
+        value: "S3", // 地址栏显示的hash值
+        id: "decision-intelligence-cluster-file-s3", // id
+        text: "S3", // 文字
+        cardType: "dec.intelligence.cluster.file.s3",
+        workRoot: false,
+    });
+
+    return items;
+});
+
+
+!(function () {
+    var LABEL_WIDTH = 107, EDITOR_WIDTH = 393;
+    var S3 = BI.inherit(BI.Widget, {
+
+        props: {
+            baseCls: "dec-cluster-ftp",
+            value: {},
+        },
+
+        _store: function () {
+            return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", {
+                value: this.options.value,
+            });
+        },
+
+        render: function () {
+            var self = this, o = this.options;
+            console.log(this.model.enablePathStyleAccess);
+
+            return {
+                type: "bi.vertical",
+                tgap: 15,
+                items: [
+                    {
+                        type: "dec.label.editor.item",
+                        textWidth: LABEL_WIDTH,
+                        editorWidth: EDITOR_WIDTH,
+                        watermark: BI.i18nText("Plugin-S3_Input"),
+                        text: BI.i18nText("Plugin-S3_End_Point"),
+                        value: this.model.endPoint,
+                        el: {
+                            disabled: !o.editable,
+                        },
+                        ref: function (_ref) {
+                            self.endPointRow = _ref;
+                        },
+                        listeners: [{
+                            eventName: BI.Editor.EVENT_CHANGE,
+                            action: function () {
+                                self.store.setEndPoint(this.getValue());
+                            },
+                        }],
+                    }, {
+                        type: "dec.label.editor.item",
+                        textWidth: LABEL_WIDTH,
+                        editorWidth: EDITOR_WIDTH,
+                        watermark: BI.i18nText("Plugin-S3_Input"),
+                        text: BI.i18nText("Plugin-S3_Region"),
+                        value: this.model.region,
+                        el: {
+                            disabled: !o.editable,
+                        },
+                        ref: function (_ref) {
+                            self.regionRow = _ref;
+                        },
+                        listeners: [{
+                            eventName: BI.Editor.EVENT_CHANGE,
+                            action: function () {
+                                self.store.setRegion(this.getValue());
+                            },
+                        }],
+                    }, {
+                        type: "dec.label.editor.item",
+                        textWidth: LABEL_WIDTH,
+                        editorWidth: EDITOR_WIDTH,
+                        watermark: BI.i18nText("Plugin-S3_Input"),
+                        text: BI.i18nText("Plugin-S3_Access_Key_Id"),
+                        value: this.model.accessKeyId,
+                        el: {
+                            disabled: !o.editable,
+                        },
+                        ref: function (_ref) {
+                            self.portRow = _ref;
+                        },
+                        listeners: [{
+                            eventName: BI.Editor.EVENT_CHANGE,
+                            action: function () {
+                                self.store.setAccessKeyId(this.getValue());
+                            },
+                        }],
+                    },
+                    {
+                        type: "dec.common.cipher.editor",
+                        textWidth: LABEL_WIDTH,
+                        editorWidth: EDITOR_WIDTH,
+                        watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"),
+                        text: BI.i18nText("Plugin-S3_Access_Key_Secret"),
+                        value: this.model.password,
+                        el: {
+                            disabled: !o.editable,
+                        },
+                        ref: function (_ref) {
+                            self.passwordRow = _ref;
+                        },
+                    },
+                    {
+                        type: "dec.label.editor.item",
+                        textWidth: LABEL_WIDTH,
+                        editorWidth: EDITOR_WIDTH,
+                        watermark: BI.i18nText("Plugin-S3_Bucket"),
+                        text: BI.i18nText("Plugin-S3_Bucket"),
+                        value: this.model.bucket,
+                        el: {
+                            disabled: !o.editable,
+                        },
+                        listeners: [{
+                            eventName: BI.Editor.EVENT_CHANGE,
+                            action: function () {
+                                self.store.setBucket(this.getValue());
+                            },
+                        }],
+                    },
+                    {
+                        type: "dec.label.editor.item",
+                        el: {
+                            disabled: !o.editable,
+                        },
+                        textWidth: LABEL_WIDTH,
+                        editorWidth: EDITOR_WIDTH,
+                        watermark: BI.i18nText("Dec-Please_Input"),
+                        text: BI.i18nText("Dec-Basic_Path"),
+                        value: this.model.workRoot,
+                        ref: function (_ref) {
+                            self.filePathRow = _ref;
+                        },
+                    }, {
+                        type: "bi.vertical_adapt",
+                        items: [{
+                            type: "bi.icon_change_button",
+                            iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font",
+                            ref: (_ref) => {
+                                this.OtherConfigButton = _ref;
+                            },
+                            handler: () => {
+                                this.store.setIsOpen(!this.model.isOpen);
+                                this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
+                            }
+                        }, {
+                            type: "bi.text_button",
+                            text: BI.i18nText('Plugin-S3_Other_Config'),
+                            handler: () => {
+                                this.store.setIsOpen(!this.model.isOpen);
+                                this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
+                            }
+                        }]
+                    }, {
+                        type: 'bi.vertical',
+                        invisible: () => !this.model.isOpen,
+                        items: [{
+                            type: "dec.label.editor.item",
+                            textWidth: LABEL_WIDTH,
+                            editorWidth: EDITOR_WIDTH,
+                            watermark: BI.i18nText("Plugin-S3_Input"),
+                            text: "PathStyleAccess",
+                            value: String(this.model.enablePathStyleAccess),
+                            ref: function (_ref) {
+                                self.enablePathStyleAccessRow = _ref;
+                            },
+                            el: {
+                                disabled: !o.editable,
+                            },
+                            listeners: [{
+                                eventName: BI.Editor.EVENT_CHANGE,
+                                action: function () {
+                                    self.store.setEnablePathStyleAccess(this.getValue());
+                                }
+                            }]
+                        }, {
+                            type: "dec.label.editor.item",
+                            textWidth: LABEL_WIDTH,
+                            editorWidth: EDITOR_WIDTH,
+                            watermark: BI.i18nText("Plugin-S3_Input"),
+                            text: "SignerOverride",
+                            value: this.model.signerOverride,
+                            el: {
+                                disabled: !o.editable,
+                            },
+                            tgap: 15,
+                            listeners: [{
+                                eventName: BI.Editor.EVENT_CHANGE,
+                                action: function () {
+                                    self.store.setSignerOverride(this.getValue());
+                                }
+                            }]
+                        }],
+                    }
+                ]
+            };
+        },
+
+        getValue: function () {
+            var enablePathStyleAccess = false;
+            if (this.model.enablePathStyleAccess === 'true') {
+                enablePathStyleAccess = true;
+            }
+
+            return {
+                endPoint: this.model.endPoint,
+                region: this.model.region,
+                accessKeyId: this.model.accessKeyId,
+                password: this.passwordRow.getCompleteCipher(),
+                bucket: this.model.bucket,
+                workRoot: this.filePathRow.getValue(),
+                enablePathStyleAccess,
+                signerOverride: this.model.signerOverride,
+            };
+        },
+
+        validation: function () {
+            var valid = true;
+            var path = this.filePathRow.getValue();
+            if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) {
+                this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text"));
+                valid = false;
+            }
+            if (BI.startWith(path, "/") || !BI.endWith(path, "/")) {
+                this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash"));
+                valid = false;
+            }
+            if (!BI.isKey(path)) {
+                this.filePathRow.showError(BI.i18nText("Dec-Error_Null"));
+                valid = false;
+            }
+            if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) {
+                this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip"));
+                valid = false;
+            }
+
+            return valid;
+        },
+
+    });
+    BI.shortcut("dec.intelligence.cluster.file.s3", S3);
+}());
+
+
+!(function () {
+    var Model = BI.inherit(Fix.Model, {
+
+        state: function () {
+            var val = this.options.value;
+            console.log(this.options.value);
+
+            return {
+                endPoint: val.endPoint,
+                region: val.region,
+                accessKeyId: val.accessKeyId,
+                password: val.password,
+                bucket: val.bucket,
+                workRoot: val.workRoot,
+                isOpen: false,
+                enablePathStyleAccess: val.enablePathStyleAccess,
+                signerOverride: val.signerOverride,
+            };
+        },
+
+        computed: {
+            encodingArray: function () {
+                return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) {
+                    return {
+                        value: v,
+                    };
+                });
+            },
+        },
+
+        actions: {
+            setEndPoint: function (v) {
+                this.model.endPoint = v;
+            },
+
+            setRegion: function (v) {
+                this.model.region = v;
+            },
+
+            setAccessKeyId: function (v) {
+                this.model.accessKeyId = v;
+            },
+
+            setBucket: function (v) {
+                this.model.bucket = v;
+            },
+
+            setEnablePathStyleAccess: function (v) {
+                this.model.enablePathStyleAccess = v;
+            },
+
+            setSignerOverride: function (v) {
+                this.model.signerOverride = v;
+            },
+
+            setIsOpen: function (v) {
+                this.model.isOpen = v;
+            },
+        },
+    });
+    BI.model("dec.model.intelligence.cluster.file.s3", Model);
+}());

front/gulpfile.js

@@ -0,0 +1,10 @@
+const gulp = require('gulp');
+const uglify = require('gulp-uglify');
+
+function build() {
+    return gulp.src('./bundle.js')
+        .pipe(uglify())
+        .pipe(gulp.dest('../src/main/resources/com/fanruan/fs/s3/repository/web/js'));
+}
+
+exports.default = build;

front/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "front",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "scripts": {
+    "build": "gulp"
+  },
+  "dependencies": {
+    "gulp": "^4.0.2",
+    "gulp-uglify": "^3.0.2"
+  }
+}

plugin.xml

@@ -5,12 +5,13 @@
     <main-package>com.fanruan.fs</main-package>
     <active>yes</active>
     <hidden>no</hidden>
-    <version>1.3.6</version>
+    <version>1.3.8</version>
     <env-version>11.0~11.0</env-version>
-    <jartime>2021-08-30</jartime>
+    <jartime>2023-03-14</jartime>
     <vendor>richie</vendor>
     <description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
     <change-notes><![CDATA[
+        [2023-03-28]第三方组件升级。 <br/>
         [2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/>
         [2022-09-22]第三方组件升级。 <br/>
         [2022-07-25]修复http无法连接，而https和缺省的时候正常<br/>

readme.md

@@ -2,4 +2,11 @@
 
 能够使用支持S3协议的云存储，作为文件服务器，安装好插件后在决策平台的智能运维->集群配置->文件一致性设置选项下，按下图设置即可：
 
-![ui](screenshots/s3.png)
+![ui](screenshots/s3.png)
+
+# 前台gulp打包：
+进入front文件夹：`cd front`
+
+安装依赖：`yarn install`
+
+打包：`yarn build`

src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java

@@ -13,6 +13,8 @@ import com.fr.io.context.info.RepositoryProfile;
 import com.fr.io.repository.ResourceRepository;
 import com.fr.stable.StringUtils;
 
+import static com.fanruan.fs.s3.repository.core.S3ResourceRepository.HTTP;
+
 /**
  * @author richie
  * @version 10.0
@@ -42,16 +44,19 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory<S3Config> {
             BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword());
             AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
                     .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion()))
-                    .withCredentials(new AWSStaticCredentialsProvider(credentials));
+                    .withCredentials(new AWSStaticCredentialsProvider(credentials)).disableChunkedEncoding();
             if (config.isEnablePathStyleAccess()) {
                 amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
             }
+            ClientConfiguration clientConfiguration = new ClientConfiguration();
             if (StringUtils.isNotEmpty(config.getSignerOverride())) {
-                ClientConfiguration clientConfiguration = new ClientConfiguration();
                 clientConfiguration.setSignerOverride(config.getSignerOverride());
+            }
+            LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
+            if (config.getEndPoint().startsWith(HTTP)) {
                 clientConfiguration.setProtocol(Protocol.HTTP);
-                amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
             }
+            amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
             AmazonS3 s3 = amazonS3ClientBuilder.build();
             s3.listObjects(config.getBucket());
         } catch (Exception e) {

src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@@ -7,6 +7,7 @@ import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
@@ -36,8 +37,12 @@ import java.util.List;
  */
 public class S3ResourceRepository extends BaseResourceRepository {
 
+    private static final int PAGE_SIZE = 1000;
+
     private static final String DELIMITER = "/";
 
+    public static final String HTTP = "http:";
+
     private final AmazonS3 s3;
     private final String bucket;
 
@@ -55,7 +60,10 @@ public class S3ResourceRepository extends BaseResourceRepository {
         if (StringUtils.isNotEmpty(config.getSignerOverride())) {
             clientConfiguration.setSignerOverride(config.getSignerOverride());
         }
-        clientConfiguration.setProtocol(Protocol.HTTP);
+        LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
+        if (config.getEndPoint().startsWith(HTTP)) {
+            clientConfiguration.setProtocol(Protocol.HTTP);
+        }
         amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
         this.s3 = amazonS3ClientBuilder.build();
         this.bucket = config.getBucket();
@@ -188,32 +196,49 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     @Override
     public boolean delete(String path) {
-
-        s3.deleteObject(bucket, path);
-        String prefix = path;
-        if (!path.endsWith(DELIMITER)) {
-            prefix = path + DELIMITER;
-        }
         try {
-            ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
-                    .withPrefix(prefix).withDelimiter(DELIMITER);
-            ObjectListing objectListing = s3.listObjects(listObjectsRequest);
-            for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
-                String key = summary.getKey();
-                if (!key.endsWith(DELIMITER)) {
-                    s3.deleteObject(bucket, key);
+            if (isDirectory(path)) {
+                if (!path.endsWith(DELIMITER)) {
+                    path += DELIMITER;
                 }
-            }
-            for (String pre : objectListing.getCommonPrefixes()) {
-                delete(pre);
+                deleteDirectory(path);
+            } else {
+                deleteFile(path);
             }
         } catch (Exception e) {
-            LogKit.error(e.getMessage(), e);
+            LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage());
+            return false;
         }
-        s3.deleteObject(bucket, prefix);
         return true;
     }
 
+    private void deleteFile(String path) throws Exception {
+        s3.deleteObject(bucket, path);
+    }
+
+    private void deleteDirectory(String path) throws Exception {
+        List<String> files = new ArrayList<>();
+        for (FineFileEntry fineFileEntry : listEntry(path)) {
+            if (fineFileEntry.isDirectory()) {
+                deleteDirectory(fineFileEntry.getPath());
+            } else {
+                files.add(fineFileEntry.getPath());
+            }
+        }
+        deleteFiles(files);
+        deleteFile(path);
+    }
+
+    private void deleteFiles(List<String> paths) throws Exception {
+        //最多只能同时指定1000个key
+        for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) {
+            int toIndex = Math.min(i + PAGE_SIZE, paths.size());
+            DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket)
+                    .withKeys(paths.subList(i, toIndex).toArray(new String[0]));
+            s3.deleteObjects(deleteObjectsRequest);
+        }
+    }
+
     @Override
     public boolean exist(String path) {
         return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path);
@@ -268,34 +293,28 @@ public class S3ResourceRepository extends BaseResourceRepository {
         ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                 .withPrefix(dir).withDelimiter(DELIMITER);
         ObjectListing objectListing = s3.listObjects(listObjectsRequest);
-        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
+        collectFileName(dir, result, objectListing);
+        while (objectListing.isTruncated()) {
+            objectListing = s3.listNextBatchOfObjects(objectListing);
+            collectFileName(dir, result, objectListing);
+        }
+        if (filter != null) {
+            return result.stream().filter(filter::accept).toArray(String[]::new);
+        }
+        return result.toArray(new String[0]);
+    }
 
+    private void collectFileName(String dir, List<String> result, ObjectListing objectListing) {
+        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
             String key = objectSummary.getKey();
             if (StringKit.equals(key, dir)) {
                 continue;
             }
-            String[] arr = key.split(DELIMITER);
-            String name = arr[arr.length - 1];
-            if (filter == null) {
-                result.add(name);
-            } else {
-                if (filter.accept(name)) {
-                    result.add(name);
-                }
-            }
+            result.add(key.substring(key.lastIndexOf(DELIMITER) + 1));
         }
         for (String prefix : objectListing.getCommonPrefixes()) {
-            String[] arr = prefix.split(DELIMITER);
-            String name = arr[arr.length - 1] + DELIMITER;
-            if (filter == null) {
-                result.add(name);
-            } else {
-                if (filter.accept(name)) {
-                    result.add(name);
-                }
-            }
+            result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1));
         }
-        return result.toArray(new String[0]);
     }
 
     @Override
@@ -401,27 +420,4 @@ public class S3ResourceRepository extends BaseResourceRepository {
         return entry;
     }
 
-    /**
-     * 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt}
-     * 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]}
-     */
-    private List<String> getAllNecessaryPath(String path) {
-
-        // 获取所有path路径及其所有父路径
-        List<String> allPath = new ArrayList<>();
-        allPath.add(path);
-        int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
-        // 以/结尾的先把末尾的/去掉
-        if (lastIdxOfDelimiter == path.length() - 1) {
-            path = path.substring(0, lastIdxOfDelimiter);
-            lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
-        }
-        while (lastIdxOfDelimiter > 0) {
-            allPath.add(path.substring(0, lastIdxOfDelimiter + 1));
-            path = path.substring(0, lastIdxOfDelimiter);
-            lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
-        }
-        return allPath;
-    }
-
 }