diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 093dc20..4cb90ec 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -7,6 +7,7 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; @@ -36,6 +37,8 @@ import java.util.List; */ public class S3ResourceRepository extends BaseResourceRepository { + private static final int PAGE_SIZE = 1000; + private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -193,32 +196,49 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public boolean delete(String path) { - - s3.deleteObject(bucket, path); - String prefix = path; - if (!path.endsWith(DELIMITER)) { - prefix = path + DELIMITER; - } try { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) - .withPrefix(prefix).withDelimiter(DELIMITER); - ObjectListing objectListing = s3.listObjects(listObjectsRequest); - for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - String key = summary.getKey(); - if (!key.endsWith(DELIMITER)) { - s3.deleteObject(bucket, key); + if (isDirectory(path)) { + if (!path.endsWith(DELIMITER)) { + path += DELIMITER; } - } - for (String pre : objectListing.getCommonPrefixes()) { - delete(pre); + deleteDirectory(path); + } else { + deleteFile(path); } } catch (Exception e) { - LogKit.error(e.getMessage(), e); + LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage()); + return false; } - s3.deleteObject(bucket, prefix); return true; } + private void deleteFile(String path) throws Exception { + s3.deleteObject(bucket, path); + } + + private void deleteDirectory(String path) throws Exception { + List files = new ArrayList<>(); + for (FineFileEntry fineFileEntry : listEntry(path)) { + if (fineFileEntry.isDirectory()) { + deleteDirectory(fineFileEntry.getPath()); + } else { + files.add(fineFileEntry.getPath()); + } + } + deleteFiles(files); + deleteFile(path); + } + + private void deleteFiles(List paths) throws Exception { + //最多只能同时指定1000个key + for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) { + int toIndex = Math.min(i + PAGE_SIZE, paths.size()); + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket) + .withKeys(paths.subList(i, toIndex).toArray(new String[0])); + s3.deleteObjects(deleteObjectsRequest); + } + } + @Override public boolean exist(String path) { return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path); @@ -273,34 +293,28 @@ public class S3ResourceRepository extends BaseResourceRepository { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) .withPrefix(dir).withDelimiter(DELIMITER); ObjectListing objectListing = s3.listObjects(listObjectsRequest); - for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + collectFileName(dir, result, objectListing); + while (objectListing.isTruncated()) { + objectListing = s3.listNextBatchOfObjects(objectListing); + collectFileName(dir, result, objectListing); + } + if (filter != null) { + return result.stream().filter(filter::accept).toArray(String[]::new); + } + return result.toArray(new String[0]); + } + private void collectFileName(String dir, List result, ObjectListing objectListing) { + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String key = objectSummary.getKey(); if (StringKit.equals(key, dir)) { continue; } - String[] arr = key.split(DELIMITER); - String name = arr[arr.length - 1]; - if (filter == null) { - result.add(name); - } else { - if (filter.accept(name)) { - result.add(name); - } - } + result.add(key.substring(key.lastIndexOf(DELIMITER) + 1)); } for (String prefix : objectListing.getCommonPrefixes()) { - String[] arr = prefix.split(DELIMITER); - String name = arr[arr.length - 1] + DELIMITER; - if (filter == null) { - result.add(name); - } else { - if (filter.accept(name)) { - result.add(name); - } - } + result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1)); } - return result.toArray(new String[0]); } @Override @@ -406,27 +420,4 @@ public class S3ResourceRepository extends BaseResourceRepository { return entry; } - /** - * 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt} - * 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]} - */ - private List getAllNecessaryPath(String path) { - - // 获取所有path路径及其所有父路径 - List allPath = new ArrayList<>(); - allPath.add(path); - int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - // 以/结尾的先把末尾的/去掉 - if (lastIdxOfDelimiter == path.length() - 1) { - path = path.substring(0, lastIdxOfDelimiter); - lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - } - while (lastIdxOfDelimiter > 0) { - allPath.add(path.substring(0, lastIdxOfDelimiter + 1)); - path = path.substring(0, lastIdxOfDelimiter); - lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - } - return allPath; - } - }