diff --git a/plugin.xml b/plugin.xml index 0821aa9..9883cd0 100644 --- a/plugin.xml +++ b/plugin.xml @@ -5,12 +5,13 @@ com.fanruan.fs yes no - 1.3.9 + 1.4.0 10.0~10.0 2023-03-14 richie [2023-06-30]修复默认配置获取错误的问题,过滤有问题的路径。
[2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 21e5d7b..f82d7ad 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -7,25 +7,33 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; +import com.fr.io.utils.ResourceIOUtils; import com.fr.stable.Filter; import com.fr.stable.StringUtils; import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; @@ -41,6 +49,10 @@ public class S3ResourceRepository extends BaseResourceRepository { private static final int PAGE_SIZE = 1000; + private static final int PART_SIZE = 5 * 1024 * 1024; + + private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE; + private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -67,7 +79,6 @@ public class S3ResourceRepository extends BaseResourceRepository { if (config.getEndPoint().startsWith(HTTP)) { clientConfiguration.setProtocol(Protocol.HTTP); } - clientConfiguration.setProtocol(Protocol.HTTP); amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); this.s3 = amazonS3ClientBuilder.build(); this.bucket = config.getBucket(); @@ -142,22 +153,71 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public void write(String path, byte[] data) { - ObjectMetadata metadata; - try { - metadata = s3.getObjectMetadata(bucket, path); - } catch (Exception e) { - metadata = new ObjectMetadata(); - String mimeType = URLConnection.guessContentTypeFromName(path); - if (mimeType != null) { - metadata.setContentType(mimeType); + int length = data.length; + if (length > MULTIPART_UPLOAD_LIMIT) { + multipartUpload(path, new ByteArrayInputStream(data)); + } else { + ObjectMetadata metadata; + try { + metadata = s3.getObjectMetadata(bucket, path); + } catch (Exception e) { + metadata = new ObjectMetadata(); + String mimeType = URLConnection.guessContentTypeFromName(path); + if (mimeType != null) { + metadata.setContentType(mimeType); + } } + if (metadata != null) { + metadata.setContentLength(length); + } + s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); } - if (metadata != null) { - metadata.setContentLength(data.length); - } - s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); } + @Override + public void write(String path, InputStream inputStream) throws ResourceIOException { + multipartUpload(path, inputStream); + } + + private void multipartUpload(String path, InputStream inputStream) { + try { + // Step 1: 初始化分片上传 + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path); + InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + + // Step 2: 分片上传文件 + List partETags = new ArrayList<>(); + byte[] buffer = new byte[PART_SIZE]; + int bytesRead; + int partNumber = 1; + + while ((bytesRead = inputStream.read(buffer)) > 0) { + // 创建上传请求 + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucket) + .withKey(path) + .withUploadId(uploadId) + .withPartNumber(partNumber) + .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead)) + .withPartSize(bytesRead); + + // 上传分片 + UploadPartResult uploadResult = s3.uploadPart(uploadRequest); + partETags.add(uploadResult.getPartETag()); + + partNumber++; + } + + // Step 3: 完成分片上传 + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags); + s3.completeMultipartUpload(compRequest); + } catch (IOException e) { + throw new ResourceIOException(e); + } finally { + ResourceIOUtils.close(inputStream); + } + } @Override public boolean createFile(String path) { @@ -331,19 +391,16 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public boolean isDirectory(String path) { - - if (path.endsWith(DELIMITER) && exist(path)) { - return true; - } - ObjectListing listing = s3.listObjects(bucket, path); - if (listing.getObjectSummaries().isEmpty()) { - return false; - } - if (listing.getObjectSummaries().size() > 1) { - return true; + if (path.endsWith(DELIMITER)) { + return exist(path); } else { - S3ObjectSummary summary = listing.getObjectSummaries().get(0); - return !StringKit.equals(listing.getPrefix(), summary.getKey()); + ObjectListing listing = s3.listObjects(bucket, path); + List objectSummaries = listing.getObjectSummaries(); + if (objectSummaries.isEmpty()) { + return false; + } + String dirFormat = path + DELIMITER; + return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat)); } }