From 8d67ac17a2ec122724ff6ff5e51d9c28c12a818c Mon Sep 17 00:00:00 2001 From: Afly Date: Tue, 23 May 2023 17:45:03 +0800 Subject: [PATCH 1/4] =?UTF-8?q?REPORT-96529=20fix:=20list=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=95=B0=E9=87=8F=E9=99=90=E5=88=B6;=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E6=8E=A5=E5=8F=A3=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3ResourceRepository.java | 113 ++++++++---------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index d96756e..5a21574 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -7,6 +7,7 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; @@ -36,6 +37,8 @@ import java.util.List; */ public class S3ResourceRepository extends BaseResourceRepository { + private static final int PAGE_SIZE = 1000; + private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -193,32 +196,49 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public boolean delete(String path) { - - s3.deleteObject(bucket, path); - String prefix = path; - if (!path.endsWith(DELIMITER)) { - prefix = path + DELIMITER; - } try { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) - .withPrefix(prefix).withDelimiter(DELIMITER); - ObjectListing objectListing = s3.listObjects(listObjectsRequest); - for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - String key = summary.getKey(); - if (!key.endsWith(DELIMITER)) { - s3.deleteObject(bucket, key); + if (isDirectory(path)) { + if (!path.endsWith(DELIMITER)) { + path += DELIMITER; } - } - for (String pre : objectListing.getCommonPrefixes()) { - delete(pre); + deleteDirectory(path); + } else { + deleteFile(path); } } catch (Exception e) { - LogKit.error(e.getMessage(), e); + LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage()); + return false; } - s3.deleteObject(bucket, prefix); return true; } + private void deleteFile(String path) throws Exception { + s3.deleteObject(bucket, path); + } + + private void deleteDirectory(String path) throws Exception { + List files = new ArrayList<>(); + for (FineFileEntry fineFileEntry : listEntry(path)) { + if (fineFileEntry.isDirectory()) { + deleteDirectory(fineFileEntry.getPath()); + } else { + files.add(fineFileEntry.getPath()); + } + } + deleteFiles(files); + deleteFile(path); + } + + private void deleteFiles(List paths) throws Exception { + //最多只能同时指定1000个key + for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) { + int toIndex = Math.min(i + PAGE_SIZE, paths.size()); + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket) + .withKeys(paths.subList(i, toIndex).toArray(new String[0])); + s3.deleteObjects(deleteObjectsRequest); + } + } + @Override public boolean exist(String path) { return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path); @@ -273,34 +293,28 @@ public class S3ResourceRepository extends BaseResourceRepository { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) .withPrefix(dir).withDelimiter(DELIMITER); ObjectListing objectListing = s3.listObjects(listObjectsRequest); - for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + collectFileName(dir, result, objectListing); + while (objectListing.isTruncated()) { + objectListing = s3.listNextBatchOfObjects(objectListing); + collectFileName(dir, result, objectListing); + } + if (filter != null) { + return result.stream().filter(filter::accept).toArray(String[]::new); + } + return result.toArray(new String[0]); + } + private void collectFileName(String dir, List result, ObjectListing objectListing) { + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String key = objectSummary.getKey(); if (StringKit.equals(key, dir)) { continue; } - String[] arr = key.split(DELIMITER); - String name = arr[arr.length - 1]; - if (filter == null) { - result.add(name); - } else { - if (filter.accept(name)) { - result.add(name); - } - } + result.add(key.substring(key.lastIndexOf(DELIMITER) + 1)); } for (String prefix : objectListing.getCommonPrefixes()) { - String[] arr = prefix.split(DELIMITER); - String name = arr[arr.length - 1] + DELIMITER; - if (filter == null) { - result.add(name); - } else { - if (filter.accept(name)) { - result.add(name); - } - } + result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1)); } - return result.toArray(new String[0]); } @Override @@ -406,27 +420,4 @@ public class S3ResourceRepository extends BaseResourceRepository { return entry; } - /** - * 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt} - * 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]} - */ - private List getAllNecessaryPath(String path) { - - // 获取所有path路径及其所有父路径 - List allPath = new ArrayList<>(); - allPath.add(path); - int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - // 以/结尾的先把末尾的/去掉 - if (lastIdxOfDelimiter == path.length() - 1) { - path = path.substring(0, lastIdxOfDelimiter); - lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - } - while (lastIdxOfDelimiter > 0) { - allPath.add(path.substring(0, lastIdxOfDelimiter + 1)); - path = path.substring(0, lastIdxOfDelimiter); - lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - } - return allPath; - } - } From 590584026d4e054d0f72db8eaff37e0cedf668cd Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:13 +0800 Subject: [PATCH 2/4] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5=E6=B1=A0?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fanruan/fs/s3/repository/core/S3Config.java | 15 +++++++++++++++ .../s3/repository/core/S3ResourceRepository.java | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java index 5821864..976a9b8 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java @@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig { @Identifier("signerOverride") private Conf signerOverride = HolderKit.simple(StringUtils.EMPTY); + @Identifier("maxConnections") + private Conf maxConnections = HolderKit.simple(200); + @GetConfig("endPoint") public String getEndPoint() { return endPoint.get(); @@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig { this.signerOverride.set(signerOverride); } + @GetConfig("maxConnections") + public int getMaxConnections() { + return maxConnections.get(); + } + + @SetConfig("maxConnections") + public void setMaxConnections(int maxConnections) { + this.maxConnections.set(maxConnections); + } + @Override public void update(String key) { super.update(key); @@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig { this.setBucket(newConfig.getBucket()); this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess()); this.setSignerOverride(newConfig.getSignerOverride()); + this.setMaxConnections(newConfig.getMaxConnections()); } } @@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig { cloned.bucket = (Conf) bucket.clone(); cloned.enablePathStyleAccess = (Conf) enablePathStyleAccess.clone(); cloned.signerOverride = (Conf) signerOverride.clone(); + cloned.maxConnections = (Conf) maxConnections.clone(); return cloned; } } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 5a21574..ca2b05d 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -15,6 +15,7 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; @@ -56,6 +57,8 @@ public class S3ResourceRepository extends BaseResourceRepository { amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); } ClientConfiguration clientConfiguration = new ClientConfiguration(); + clientConfiguration.setMaxConnections(config.getMaxConnections()); + LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections()); if (StringUtils.isNotEmpty(config.getSignerOverride())) { clientConfiguration.setSignerOverride(config.getSignerOverride()); } @@ -128,6 +131,8 @@ public class S3ResourceRepository extends BaseResourceRepository { try { return s3.getObject(request).getObjectContent(); } catch (Exception e) { + LogKit.error("[S3] Failed to read file {}", filePath); + LogKit.error(e.getMessage(), e); return new ByteArrayInputStream(new byte[0]); } } @@ -341,6 +346,8 @@ public class S3ResourceRepository extends BaseResourceRepository { S3Object s3Object = s3.getObject(bucket, path); if (s3Object != null) { try { + //s3Object要全部读完,否则会有警告 + IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream()); return s3Object.getObjectMetadata().getLastModified().getTime(); } finally { s3Object.close(); From b1b2c507cb4e1138f90c66e93a4f57bdcd15695d Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:36 +0800 Subject: [PATCH 3/4] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5=E6=B1=A0?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fanruan/fs/s3/repository/core/S3ResourceRepository.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index ca2b05d..614edec 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -22,6 +22,7 @@ import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; import com.fr.stable.Filter; import com.fr.stable.StringUtils; +import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream; From 7d4a1f54e3d93e324e13d68aadf3265a853db257 Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 15 Jun 2023 09:58:41 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20fix:=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=BF=9E=E6=8E=A5=E5=90=8E=E9=87=8A=E6=94=BE?= =?UTF-8?q?=E8=B5=84=E6=BA=90;list=E6=8E=A5=E5=8F=A3=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E8=87=AA=E8=BA=AB=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3RepositoryFactory.java | 7 ++++++- .../repository/core/S3ResourceRepository.java | 20 ++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java index 2aecad2..5de506c 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java @@ -40,6 +40,7 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { @Override public boolean verifyConfig(S3Config config) { + AmazonS3 s3 = null; try { BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword()); AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() @@ -57,11 +58,15 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { clientConfiguration.setProtocol(Protocol.HTTP); } amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); - AmazonS3 s3 = amazonS3ClientBuilder.build(); + s3 = amazonS3ClientBuilder.build(); s3.listObjects(config.getBucket()); } catch (Exception e) { LogKit.error(e.getMessage(), e); return false; + } finally { + if (s3 != null) { + s3.shutdown(); + } } return true; } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 614edec..21e5d7b 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -99,15 +99,15 @@ public class S3ResourceRepository extends BaseResourceRepository { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) .withPrefix(dir).withDelimiter(DELIMITER); ObjectListing objectListing = s3.listObjects(listObjectsRequest); - collectFileEntry(result, objectListing); + collectFileEntry(dir, result, objectListing); while (objectListing.isTruncated()) { objectListing = s3.listNextBatchOfObjects(objectListing); - collectFileEntry(result, objectListing); + collectFileEntry(dir, result, objectListing); } return result.toArray(new FineFileEntry[0]); } - private void collectFileEntry(List result, ObjectListing objectListing) { + private void collectFileEntry(String dir, List result, ObjectListing objectListing) { for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { String key = summary.getKey(); if (!key.endsWith(DELIMITER)) { @@ -115,9 +115,11 @@ public class S3ResourceRepository extends BaseResourceRepository { } } for (String prefix : objectListing.getCommonPrefixes()) { - FineFileEntry entry = new FineFileEntry(prefix); - entry.setDirectory(true); - result.add(entry); + if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) { + FineFileEntry entry = new FineFileEntry(prefix); + entry.setDirectory(true); + result.add(entry); + } } } @@ -319,7 +321,11 @@ public class S3ResourceRepository extends BaseResourceRepository { result.add(key.substring(key.lastIndexOf(DELIMITER) + 1)); } for (String prefix : objectListing.getCommonPrefixes()) { - result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1)); + if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) { + String[] arr = prefix.split(DELIMITER); + String name = arr[arr.length - 1] + DELIMITER; + result.add(name); + } } }