diff --git a/plugin.xml b/plugin.xml
index 0821aa9..9883cd0 100644
--- a/plugin.xml
+++ b/plugin.xml
@@ -5,12 +5,13 @@
com.fanruan.fs
yes
no
- 1.3.9
+ 1.4.0
10.0~10.0
2023-03-14
richie
[2023-06-30]修复默认配置获取错误的问题,过滤有问题的路径。
[2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
index 21e5d7b..f82d7ad 100644
--- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
+++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
@@ -7,25 +7,33 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
import com.amazonaws.util.IOUtils;
import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
+import com.fr.io.utils.ResourceIOUtils;
import com.fr.stable.Filter;
import com.fr.stable.StringUtils;
import com.fr.third.org.apache.commons.io.output.NullOutputStream;
import com.fr.workspace.resource.ResourceIOException;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
@@ -41,6 +49,10 @@ public class S3ResourceRepository extends BaseResourceRepository {
private static final int PAGE_SIZE = 1000;
+ private static final int PART_SIZE = 5 * 1024 * 1024;
+
+ private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
+
private static final String DELIMITER = "/";
public static final String HTTP = "http:";
@@ -67,7 +79,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
if (config.getEndPoint().startsWith(HTTP)) {
clientConfiguration.setProtocol(Protocol.HTTP);
}
- clientConfiguration.setProtocol(Protocol.HTTP);
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
this.s3 = amazonS3ClientBuilder.build();
this.bucket = config.getBucket();
@@ -142,22 +153,71 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public void write(String path, byte[] data) {
- ObjectMetadata metadata;
- try {
- metadata = s3.getObjectMetadata(bucket, path);
- } catch (Exception e) {
- metadata = new ObjectMetadata();
- String mimeType = URLConnection.guessContentTypeFromName(path);
- if (mimeType != null) {
- metadata.setContentType(mimeType);
+ int length = data.length;
+ if (length > MULTIPART_UPLOAD_LIMIT) {
+ multipartUpload(path, new ByteArrayInputStream(data));
+ } else {
+ ObjectMetadata metadata;
+ try {
+ metadata = s3.getObjectMetadata(bucket, path);
+ } catch (Exception e) {
+ metadata = new ObjectMetadata();
+ String mimeType = URLConnection.guessContentTypeFromName(path);
+ if (mimeType != null) {
+ metadata.setContentType(mimeType);
+ }
}
+ if (metadata != null) {
+ metadata.setContentLength(length);
+ }
+ s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
- if (metadata != null) {
- metadata.setContentLength(data.length);
- }
- s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
+ @Override
+ public void write(String path, InputStream inputStream) throws ResourceIOException {
+ multipartUpload(path, inputStream);
+ }
+
+ private void multipartUpload(String path, InputStream inputStream) {
+ try {
+ // Step 1: 初始化分片上传
+ InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
+ InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
+ String uploadId = initResponse.getUploadId();
+
+ // Step 2: 分片上传文件
+ List partETags = new ArrayList<>();
+ byte[] buffer = new byte[PART_SIZE];
+ int bytesRead;
+ int partNumber = 1;
+
+ while ((bytesRead = inputStream.read(buffer)) > 0) {
+ // 创建上传请求
+ UploadPartRequest uploadRequest = new UploadPartRequest()
+ .withBucketName(bucket)
+ .withKey(path)
+ .withUploadId(uploadId)
+ .withPartNumber(partNumber)
+ .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
+ .withPartSize(bytesRead);
+
+ // 上传分片
+ UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
+ partETags.add(uploadResult.getPartETag());
+
+ partNumber++;
+ }
+
+ // Step 3: 完成分片上传
+ CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
+ s3.completeMultipartUpload(compRequest);
+ } catch (IOException e) {
+ throw new ResourceIOException(e);
+ } finally {
+ ResourceIOUtils.close(inputStream);
+ }
+ }
@Override
public boolean createFile(String path) {
@@ -331,19 +391,16 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean isDirectory(String path) {
-
- if (path.endsWith(DELIMITER) && exist(path)) {
- return true;
- }
- ObjectListing listing = s3.listObjects(bucket, path);
- if (listing.getObjectSummaries().isEmpty()) {
- return false;
- }
- if (listing.getObjectSummaries().size() > 1) {
- return true;
+ if (path.endsWith(DELIMITER)) {
+ return exist(path);
} else {
- S3ObjectSummary summary = listing.getObjectSummaries().get(0);
- return !StringKit.equals(listing.getPrefix(), summary.getKey());
+ ObjectListing listing = s3.listObjects(bucket, path);
+ List objectSummaries = listing.getObjectSummaries();
+ if (objectSummaries.isEmpty()) {
+ return false;
+ }
+ String dirFormat = path + DELIMITER;
+ return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
}
}