From f8e1e9e277fceb265ae769a749abbd1d39cd406e Mon Sep 17 00:00:00 2001 From: "seth.tian" Date: Mon, 7 Aug 2023 17:27:47 +0800 Subject: [PATCH 1/2] =?UTF-8?q?JSY-31940=20fix:=E4=BF=AE=E6=94=B9=E5=88=86?= =?UTF-8?q?=E7=89=87=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 2 +- plugin.xml | 1 + .../repository/core/S3ResourceRepository.java | 65 ++++++++++--------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/build.gradle b/build.gradle index 0d9769c..cde6ba7 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ ext { */ libPath = "$projectDir/../webroot/WEB-INF/lib" - fineVersion = '11.0-jsy-20220713' + fineVersion = '11.0-jsy-20221012' /** * 是否对插件的class进行加密保护，防止反编译 diff --git a/plugin.xml b/plugin.xml index 6f58f0e..cf4a8d2 100644 --- a/plugin.xml +++ b/plugin.xml @@ -11,6 +11,7 @@ richie [2022-07-25]修复http无法连接，而https和缺省的时候正常
[2022-06-30]新增enablePathStyleAccess、signerOverride后台配置
[2022-06-13]第三方组件升级
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index bf8fbe0..7636c86 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -6,33 +6,31 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; -import com.fr.config.utils.RandomUtils; import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; import com.fr.log.FineLoggerFactory; import com.fr.stable.Filter; import com.fr.workspace.resource.ResourceIOException; -import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; -import java.nio.file.Files; import java.util.ArrayList; import java.util.List; @@ -45,10 +43,13 @@ public class S3ResourceRepository extends BaseResourceRepository { private static final String DELIMITER = "/"; /** - * 1.5G + * 1G */ - private static final int LENGTH_LIMIT = 1610612736; - private static final String LINK_SYMBOL = "_"; + private static final int LENGTH_LIMIT = 1024 * 1024 * 1024; + /** + * 10M一个分片大小 + */ + private static final int PART_SIZE = 10 * 1024 * 1024; private final AmazonS3 readClient; private final AmazonS3 writeClient; @@ -160,26 +161,30 @@ public class S3ResourceRepository extends BaseResourceRepository { try { // 大于1.5G的,就自己算一下长度 if (inputStream.available() > LENGTH_LIMIT) { - final long l1 = System.currentTimeMillis(); - final File file = new File(l1 + LINK_SYMBOL + RandomUtils.randomId(5)); - Files.deleteIfExists(file.toPath()); - file.createNewFile(); - try (final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(file));) { - final long l = inputStream.transferTo(bufferedOutputStream); - bufferedOutputStream.flush(); - metadata.setContentLength(l); - TransferManager tm = TransferManagerBuilder.standard() - .withS3Client(this.writeClient) - .build(); - - // TransferManager processes all transfers asynchronously, - // so this call returns immediately. - Upload upload = tm.upload(this.bucket, path, file); - // Optionally, wait for the upload to finish before continuing. - upload.waitForCompletion(); - } finally { - Files.deleteIfExists(file.toPath()); + List partETags = new ArrayList<>(); + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path); + InitiateMultipartUploadResult initResponse = this.writeClient.initiateMultipartUpload(initRequest); + final String uploadId = initResponse.getUploadId(); + byte[] buffer = new byte[PART_SIZE]; + int bytesRead; + int partNumber = 1; + while ((bytesRead = inputStream.read(buffer)) > 0) { + // Create the request to upload a part. + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucket) + .withKey(path) + .withUploadId(uploadId) + .withPartNumber(partNumber) + .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead)) + .withPartSize(bytesRead); + + final UploadPartResult uploadPartResult = this.writeClient.uploadPart(uploadRequest); + partETags.add(uploadPartResult.getPartETag()); + partNumber++; } + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, + initResponse.getUploadId(), partETags); + this.writeClient.completeMultipartUpload(compRequest); } else { metadata.setContentLength(inputStream.available()); this.writeClient.putObject(this.bucket, path, inputStream, metadata); From 139ade57a97ceac338d7a17dec33978f47279148 Mon Sep 17 00:00:00 2001 From: "seth.tian" Date: Mon, 7 Aug 2023 17:36:55 +0800 Subject: [PATCH 2/2] =?UTF-8?q?JSY-31940=20update:=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=8B=E7=89=88=E6=9C=AC=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.xml b/plugin.xml index cf4a8d2..b04cb08 100644 --- a/plugin.xml +++ b/plugin.xml @@ -5,7 +5,7 @@ com.fanruan.fs yes no - 1.3.7 + 1.3.8 11.0~11.0 2021-08-30 richie