Pull request #242: REPORT-96529 fix: list接口数量限制;删除接口性能优化

Merge in PG/plugin-repository-s3 from ~AFLY/plugin-repository-s3:release/11.0 to release/11.0

* commit '23d0b22f8517824fdcc17d27e2ba2316cc000ca5':
  REPORT-96529 fix: list接口数量限制;删除接口性能优化

src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@@ -7,6 +7,7 @@ import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
@@ -36,6 +37,8 @@ import java.util.List;
  */
 public class S3ResourceRepository extends BaseResourceRepository {
 
+    private static final int PAGE_SIZE = 1000;
+
     private static final String DELIMITER = "/";
 
     public static final String HTTP = "http:";
@@ -193,30 +196,47 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     @Override
     public boolean delete(String path) {
+        try {
+            if (isDirectory(path)) {
+                if (!path.endsWith(DELIMITER)) {
+                    path += DELIMITER;
+                }
+                deleteDirectory(path);
+            } else {
+                deleteFile(path);
+            }
+        } catch (Exception e) {
+            LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage());
+            return false;
+        }
+        return true;
+    }
 
+    private void deleteFile(String path) throws Exception {
         s3.deleteObject(bucket, path);
-        String prefix = path;
-        if (!path.endsWith(DELIMITER)) {
-            prefix = path + DELIMITER;
     }
-        try {
+
-            ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
+    private void deleteDirectory(String path) throws Exception {
-                    .withPrefix(prefix).withDelimiter(DELIMITER);
+        List<String> files = new ArrayList<>();
-            ObjectListing objectListing = s3.listObjects(listObjectsRequest);
+        for (FineFileEntry fineFileEntry : listEntry(path)) {
-            for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
+            if (fineFileEntry.isDirectory()) {
-                String key = summary.getKey();
+                deleteDirectory(fineFileEntry.getPath());
-                if (!key.endsWith(DELIMITER)) {
+            } else {
-                    s3.deleteObject(bucket, key);
+                files.add(fineFileEntry.getPath());
             }
         }
-            for (String pre : objectListing.getCommonPrefixes()) {
+        deleteFiles(files);
-                delete(pre);
+        deleteFile(path);
     }
-        } catch (Exception e) {
+
-            LogKit.error(e.getMessage(), e);
+    private void deleteFiles(List<String> paths) throws Exception {
+        //最多只能同时指定1000个key
+        for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) {
+            int toIndex = Math.min(i + PAGE_SIZE, paths.size());
+            DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket)
+                    .withKeys(paths.subList(i, toIndex).toArray(new String[0]));
+            s3.deleteObjects(deleteObjectsRequest);
         }
-        s3.deleteObject(bucket, prefix);
-        return true;
     }
 
     @Override
@@ -273,34 +293,28 @@ public class S3ResourceRepository extends BaseResourceRepository {
         ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
                 .withPrefix(dir).withDelimiter(DELIMITER);
         ObjectListing objectListing = s3.listObjects(listObjectsRequest);
-        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
+        collectFileName(dir, result, objectListing);
+        while (objectListing.isTruncated()) {
+            objectListing = s3.listNextBatchOfObjects(objectListing);
+            collectFileName(dir, result, objectListing);
+        }
+        if (filter != null) {
+            return result.stream().filter(filter::accept).toArray(String[]::new);
+        }
+        return result.toArray(new String[0]);
+    }
 
+    private void collectFileName(String dir, List<String> result, ObjectListing objectListing) {
+        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
             String key = objectSummary.getKey();
             if (StringKit.equals(key, dir)) {
                 continue;
             }
-            String[] arr = key.split(DELIMITER);
+            result.add(key.substring(key.lastIndexOf(DELIMITER) + 1));
-            String name = arr[arr.length - 1];
-            if (filter == null) {
-                result.add(name);
-            } else {
-                if (filter.accept(name)) {
-                    result.add(name);
-                }
-            }
         }
         for (String prefix : objectListing.getCommonPrefixes()) {
-            String[] arr = prefix.split(DELIMITER);
+            result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1));
-            String name = arr[arr.length - 1] + DELIMITER;
-            if (filter == null) {
-                result.add(name);
-            } else {
-                if (filter.accept(name)) {
-                    result.add(name);
-                }
-            }
         }
-        return result.toArray(new String[0]);
     }
 
     @Override
@@ -406,27 +420,4 @@ public class S3ResourceRepository extends BaseResourceRepository {
         return entry;
     }
 
-    /**
-     * 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt}
-     * 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]}
-     */
-    private List<String> getAllNecessaryPath(String path) {
-
-        // 获取所有path路径及其所有父路径
-        List<String> allPath = new ArrayList<>();
-        allPath.add(path);
-        int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
-        // 以/结尾的先把末尾的/去掉
-        if (lastIdxOfDelimiter == path.length() - 1) {
-            path = path.substring(0, lastIdxOfDelimiter);
-            lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
-        }
-        while (lastIdxOfDelimiter > 0) {
-            allPath.add(path.substring(0, lastIdxOfDelimiter + 1));
-            path = path.substring(0, lastIdxOfDelimiter);
-            lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
-        }
-        return allPath;
-    }
-
 }