From 50e2a0cdf746c16d3258b029fd01f4ff4baf6d1e Mon Sep 17 00:00:00 2001
From: "seth.tian" <seth.tian@fanruan.com>
Date: Mon, 17 Apr 2023 18:54:42 +0800
Subject: [PATCH] =?UTF-8?q?JSY-28635=20fix:=E5=AF=BC=E5=87=BA=E8=B6=85?=
=?UTF-8?q?=E8=BF=872G=E7=9A=84=E6=96=87=E4=BB=B6=E5=86=99s3=E4=BC=9A?=
=?UTF-8?q?=E5=BC=82=E5=B8=B8,=E4=B8=B4=E6=97=B6=E5=A4=84=E7=90=86?=
=?UTF-8?q?=E4=B8=80=E4=B8=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../repository/core/S3ResourceRepository.java | 42 +++++++++++++++++--
1 file changed, 39 insertions(+), 3 deletions(-)
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
index 2e62af5..24ffe83 100644
--- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
+++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
@@ -13,15 +13,22 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.amazonaws.services.s3.transfer.Upload;
import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit;
+import com.fr.config.utils.RandomUtils;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.log.FineLoggerFactory;
import com.fr.stable.Filter;
import com.fr.workspace.resource.ResourceIOException;
+import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
@@ -36,6 +43,11 @@ import java.util.List;
public class S3ResourceRepository extends BaseResourceRepository {
private static final String DELIMITER = "/";
+ /**
+ * 1.5G
+ */
+ private static final int LENGTH_LIMIT = 1610612736;
+ private static final String LINK_SYMBOL = "_";
private final AmazonS3 readClient;
private final AmazonS3 writeClient;
@@ -114,7 +126,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
return readClient.getObject(request).getObjectContent();
} catch (Exception e) {
- FineLoggerFactory.getLogger().error(e, "[S3] read path {} is error!", filePath);
+ FineLoggerFactory.getLogger().warn(e, "[S3] read path {} is error!", filePath);
return new ByteArrayInputStream(new byte[0]);
}
}
@@ -145,8 +157,32 @@ public class S3ResourceRepository extends BaseResourceRepository {
} else {
ObjectMetadata metadata = new ObjectMetadata();
try {
- metadata.setContentLength(inputStream.available());
- this.writeClient.putObject(this.bucket, path, inputStream, metadata);
+ // 大于1.5G的,就自己算一下长度
+ if (inputStream.available() > LENGTH_LIMIT) {
+ final long l1 = System.currentTimeMillis();
+ final File file = new File(l1 + LINK_SYMBOL + RandomUtils.randomId(5));
+ file.deleteOnExit();
+ file.createNewFile();
+ try (final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(file));) {
+ final long l = inputStream.transferTo(bufferedOutputStream);
+ bufferedOutputStream.flush();
+ metadata.setContentLength(l);
+ TransferManager tm = TransferManagerBuilder.standard()
+ .withS3Client(this.writeClient)
+ .build();
+
+ // TransferManager processes all transfers asynchronously,
+ // so this call returns immediately.
+ Upload upload = tm.upload(this.bucket, path, file);
+ // Optionally, wait for the upload to finish before continuing.
+ upload.waitForCompletion();
+ } finally {
+ file.deleteOnExit();
+ }
+ } else {
+ metadata.setContentLength(inputStream.available());
+ this.writeClient.putObject(this.bucket, path, inputStream, metadata);
+ }
} catch (Exception e) {
FineLoggerFactory.getLogger().info("S3 inputStream.putObject error,path is {}, error message {}", path, e.getMessage());
} finally {