From d5e6992ddad73125fe6df1b43f2db752a64ab00c Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:13 +0800 Subject: [PATCH 1/2] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5=E6=B1=A0?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fanruan/fs/s3/repository/core/S3Config.java | 15 +++++++++++++++ .../s3/repository/core/S3ResourceRepository.java | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java index 5821864..976a9b8 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java @@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig { @Identifier("signerOverride") private Conf signerOverride = HolderKit.simple(StringUtils.EMPTY); + @Identifier("maxConnections") + private Conf maxConnections = HolderKit.simple(200); + @GetConfig("endPoint") public String getEndPoint() { return endPoint.get(); @@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig { this.signerOverride.set(signerOverride); } + @GetConfig("maxConnections") + public int getMaxConnections() { + return maxConnections.get(); + } + + @SetConfig("maxConnections") + public void setMaxConnections(int maxConnections) { + this.maxConnections.set(maxConnections); + } + @Override public void update(String key) { super.update(key); @@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig { this.setBucket(newConfig.getBucket()); this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess()); this.setSignerOverride(newConfig.getSignerOverride()); + this.setMaxConnections(newConfig.getMaxConnections()); } } @@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig { cloned.bucket = (Conf) bucket.clone(); cloned.enablePathStyleAccess = (Conf) enablePathStyleAccess.clone(); cloned.signerOverride = (Conf) signerOverride.clone(); + cloned.maxConnections = (Conf) maxConnections.clone(); return cloned; } } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 4cb90ec..bfd7c47 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -15,6 +15,7 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; @@ -57,6 +58,8 @@ public class S3ResourceRepository extends BaseResourceRepository { amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); } ClientConfiguration clientConfiguration = new ClientConfiguration(); + clientConfiguration.setMaxConnections(config.getMaxConnections()); + LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections()); if (StringUtils.isNotEmpty(config.getSignerOverride())) { clientConfiguration.setSignerOverride(config.getSignerOverride()); } @@ -128,6 +131,8 @@ public class S3ResourceRepository extends BaseResourceRepository { try { return s3.getObject(request).getObjectContent(); } catch (Exception e) { + LogKit.error("[S3] Failed to read file {}", filePath); + LogKit.error(e.getMessage(), e); return new ByteArrayInputStream(new byte[0]); } } @@ -341,6 +346,8 @@ public class S3ResourceRepository extends BaseResourceRepository { S3Object s3Object = s3.getObject(bucket, path); if (s3Object != null) { try { + //s3Object要全部读完，否则会有警告 + IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream()); return s3Object.getObjectMetadata().getLastModified().getTime(); } finally { s3Object.close(); From 974c7e2b4265a55ec62574f7ef10942a3680ea17 Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:36 +0800 Subject: [PATCH 2/2] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5=E6=B1=A0?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fanruan/fs/s3/repository/core/S3ResourceRepository.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index bfd7c47..0f7b77a 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -22,6 +22,7 @@ import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; import com.fr.stable.Filter; import com.fr.stable.StringUtils; +import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream;