Pull request #87: REPORT-88094 fix: 文件数量较多时，listEntry结果不全；优化写文件性能

Merge in PG/plugin-repository-s3 from ~AFLY/plugin-repository-s3:release/10.0 to release/10.0

* commit '8f1227e659711c2c53df8afd4ab5f7e1ff13f404':
  REPORT-88094 fix: change-notes
  REPORT-88094 fix: 文件数量较多时，listEntry结果不全
  无jira任务 perf: 优化写文件性能

plugin.xml

@@ -5,12 +5,13 @@
     <main-package>com.fanruan.fs</main-package>
     <active>yes</active>
     <hidden>no</hidden>
-    <version>1.3.5</version>
+    <version>1.3.6</version>
     <env-version>10.0~10.0</env-version>
     <jartime>2021-03-11</jartime>
     <vendor>richie</vendor>
     <description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
     <change-notes><![CDATA[
+        [2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/>
         [2022-09-22]第三方组件升级。 <br/>
         [2022-07-25]修复http无法连接，而https和缺省的时候正常<br/>
         [2022-06-30]新增enablePathStyleAccess、signerOverride后台配置<br/>

src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@@ -87,6 +87,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
         ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                 .withPrefix(dir).withDelimiter(DELIMITER);
         ObjectListing objectListing = s3.listObjects(listObjectsRequest);
+        collectFileEntry(result, objectListing);
+        while (objectListing.isTruncated()) {
+            objectListing = s3.listNextBatchOfObjects(objectListing);
+            collectFileEntry(result, objectListing);
+        }
+        return result.toArray(new FineFileEntry[0]);
+    }
+
+    private void collectFileEntry(List<FineFileEntry> result, ObjectListing objectListing) {
         for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
             String key = summary.getKey();
             if (!key.endsWith(DELIMITER)) {
@@ -98,8 +107,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
             entry.setDirectory(true);
             result.add(entry);
         }
-
-        return result.toArray(new FineFileEntry[0]);
     }
 
     @Override
@@ -119,17 +126,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     @Override
     public void write(String path, byte[] data) {
-
-        List<String> paths = getAllNecessaryPath(path);
-        for (int i = paths.size() - 1; i > 0; i--) {
-            String parent = paths.get(i);
-            PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
-            try {
-                s3.putObject(req);
-            } catch (Exception e) {
-                LogKit.error("[S3] Failed to create parent path {}", parent);
-            }
-        }
         ObjectMetadata metadata;
         try {
             metadata = s3.getObjectMetadata(bucket, path);
@@ -150,16 +146,16 @@ public class S3ResourceRepository extends BaseResourceRepository {
     @Override
     public boolean createFile(String path) {
 
-        List<String> paths = getAllNecessaryPath(path);
+        String parent = path.substring(0, path.lastIndexOf(DELIMITER) + 1);
-        for (int i = paths.size() - 1; i >= 0; i--) {
+        if (!dirExist(parent)) {
-            String parent = paths.get(i);
+            createDirectory(parent);
-            PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
+        }
-            try {
+        PutObjectRequest req = new PutObjectRequest(bucket, path, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
-                s3.putObject(req);
+        try {
-            } catch (Exception e) {
+            s3.putObject(req);
-                LogKit.error("[S3] Failed to create parent path {}", parent);
+        } catch (Exception e) {
-                return false;
+            LogKit.error("[S3] Failed to create file path {}", path);
-            }
+            return false;
         }
         return true;
     }
@@ -170,17 +166,23 @@ public class S3ResourceRepository extends BaseResourceRepository {
         if (!path.endsWith(DELIMITER)) {
             path += DELIMITER;
         }
-        List<String> paths = getAllNecessaryPath(path);
+        String temp = path.substring(0, path.length() - 1);
-        for (int i = paths.size() - 1; i >= 0; i--) {
+        String parent = temp.substring(0, temp.lastIndexOf(DELIMITER) + 1);
-            String parent = paths.get(i);
+        if (StringUtils.isEmpty(parent)) {
-            PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
+            return true;
-            try {
+        }
-                s3.putObject(req);
+        if (!dirExist(parent)) {
-            } catch (Exception e) {
+            if (!createDirectory(parent)) {
-                LogKit.error("[S3] Failed to create parent path {}", parent);
                 return false;
             }
         }
+        PutObjectRequest req = new PutObjectRequest(bucket, path, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
+        try {
+            s3.putObject(req);
+        } catch (Exception e) {
+            LogKit.error("[S3] Failed to create parent path {}", path);
+            return false;
+        }
         return true;
     }
 
@@ -214,13 +216,49 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     @Override
     public boolean exist(String path) {
+        return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path);
+    }
+
+    private boolean fileExist(String path) {
         try {
-            return s3.doesObjectExist(bucket, path) || s3.doesObjectExist(bucket, path + DELIMITER);
+            return s3.doesObjectExist(bucket, path);
         } catch (Exception e) {
             return false;
         }
     }
 
+    private boolean dirExist(String path) {
+        if (StringUtils.equals(path, DELIMITER)) {
+            return true;
+        }
+        if (!path.endsWith(DELIMITER)) {
+            path += DELIMITER;
+        }
+        return fileExist(path);
+    }
+
+    /**
+     * 如果存在文件创建了，但是其父目录没有创建的场景，为其递归创建对象，返回创建结果
+     */
+    private boolean isParentPathAbsent(String path) {
+        if (path.startsWith(DELIMITER)) {
+            path = path.substring(1);
+        }
+        if (path.endsWith(DELIMITER)) {
+            path = path.substring(0, path.length() - 1);
+        }
+        ObjectListing objectListing = s3.listObjects(
+                new ListObjectsRequest()
+                        .withBucketName(bucket)
+                        .withPrefix(path));
+        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
+            if (summary.getKey().startsWith(path + DELIMITER)) {
+                return createDirectory(path);
+            }
+        }
+        return false;
+    }
+
     @Override
     public String[] list(String dir, final Filter<String> filter) {
         List<String> result = new ArrayList<>();