Pull request #402: JSY-31940 fix:修改分片上传

Merge in PG/plugin-repository-s3 from ~SETH.TIAN/plugin-repository-s3:persist/jsy-11.0 to persist/jsy-11.0

* commit '139ade57a97ceac338d7a17dec33978f47279148':
  JSY-31940 update:修改下版本号
  JSY-31940 fix:修改分片上传

build.gradle

@@ -10,7 +10,7 @@ ext {
      */
     libPath = "$projectDir/../webroot/WEB-INF/lib"
 
-    fineVersion = '11.0-jsy-20220713'
+    fineVersion = '11.0-jsy-20221012'
 
     /**
      * 是否对插件的class进行加密保护，防止反编译

plugin.xml

@@ -5,12 +5,13 @@
     <main-package>com.fanruan.fs</main-package>
     <active>yes</active>
     <hidden>no</hidden>
-    <version>1.3.7</version>
+    <version>1.3.8</version>
     <env-version>11.0~11.0</env-version>
     <jartime>2021-08-30</jartime>
     <vendor>richie</vendor>
     <description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
     <change-notes><![CDATA[
+        [2023-08-07]修改分片上传方法<br/>
         [2022-07-25]修复http无法连接，而https和缺省的时候正常<br/>
         [2022-06-30]新增enablePathStyleAccess、signerOverride后台配置<br/>
         [2022-06-13]第三方组件升级<br/>

src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@@ -6,33 +6,31 @@ import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.services.s3.transfer.Upload;
 import com.fanruan.api.log.LogKit;
 import com.fanruan.api.util.StringKit;
-import com.fr.config.utils.RandomUtils;
 import com.fr.io.repository.FineFileEntry;
 import com.fr.io.repository.base.BaseResourceRepository;
 import com.fr.log.FineLoggerFactory;
 import com.fr.stable.Filter;
 import com.fr.workspace.resource.ResourceIOException;
 
-import java.io.BufferedOutputStream;
 import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.net.URL;
 import java.net.URLConnection;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -45,10 +43,13 @@ public class S3ResourceRepository extends BaseResourceRepository {
 
     private static final String DELIMITER = "/";
     /**
-     * 1.5G
+     * 1G
      */
-    private static final int LENGTH_LIMIT = 1610612736;
+    private static final int LENGTH_LIMIT = 1024 * 1024 * 1024;
-    private static final String LINK_SYMBOL = "_";
+    /**
+     * 10M一个分片大小
+     */
+    private static final int PART_SIZE = 10 * 1024 * 1024;
 
     private final AmazonS3 readClient;
     private final AmazonS3 writeClient;
@@ -160,26 +161,30 @@ public class S3ResourceRepository extends BaseResourceRepository {
             try {
                 // 大于1.5G的,就自己算一下长度
                 if (inputStream.available() > LENGTH_LIMIT) {
-                    final long l1 = System.currentTimeMillis();
+                    List<PartETag> partETags = new ArrayList<>();
-                    final File file = new File(l1 + LINK_SYMBOL + RandomUtils.randomId(5));
+                    InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
-                    Files.deleteIfExists(file.toPath());
+                    InitiateMultipartUploadResult initResponse = this.writeClient.initiateMultipartUpload(initRequest);
-                    file.createNewFile();
+                    final String uploadId = initResponse.getUploadId();
-                    try (final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(file));) {
+                    byte[] buffer = new byte[PART_SIZE];
-                        final long l = inputStream.transferTo(bufferedOutputStream);
+                    int bytesRead;
-                        bufferedOutputStream.flush();
+                    int partNumber = 1;
-                        metadata.setContentLength(l);
+                    while ((bytesRead = inputStream.read(buffer)) > 0) {
-                        TransferManager tm = TransferManagerBuilder.standard()
+                        // Create the request to upload a part.
-                                .withS3Client(this.writeClient)
+                        UploadPartRequest uploadRequest = new UploadPartRequest()
-                                .build();
+                                .withBucketName(bucket)
-
+                                .withKey(path)
-                        // TransferManager processes all transfers asynchronously,
+                                .withUploadId(uploadId)
-                        // so this call returns immediately.
+                                .withPartNumber(partNumber)
-                        Upload upload = tm.upload(this.bucket, path, file);
+                                .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
-                        // Optionally, wait for the upload to finish before continuing.
+                                .withPartSize(bytesRead);
-                        upload.waitForCompletion();
+
-                    } finally {
+                        final UploadPartResult uploadPartResult = this.writeClient.uploadPart(uploadRequest);
-                        Files.deleteIfExists(file.toPath());
+                        partETags.add(uploadPartResult.getPartETag());
+                        partNumber++;
                     }
+                    CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path,
+                            initResponse.getUploadId(), partETags);
+                    this.writeClient.completeMultipartUpload(compRequest);
                 } else {
                     metadata.setContentLength(inputStream.available());
                     this.writeClient.putObject(this.bucket, path, inputStream, metadata);