fanruan
/
plugin-repository-s3
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

JSY-31940 fix:修改分片上传

persist/jsy-11.0^2
seth.tian 2 years ago
parent
6555c26f52
commit
f8e1e9e277
3 changed files with 37 additions and 31 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      build.gradle
  2. 1
      plugin.xml
  3. 65
      src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

2
build.gradle
Unescape View File

@ -10,7 +10,7 @@ ext {
*/
libPath = "$projectDir/../webroot/WEB-INF/lib"
fineVersion = '11.0-jsy-20220713'
fineVersion = '11.0-jsy-20221012'
/**
* 是否对插件的class进行加密保护，防止反编译

1
plugin.xml
Unescape View File

@ -11,6 +11,7 @@
<vendor>richie</vendor>
<description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
<change-notes><![CDATA[
[2023-08-07]修改分片上传方法<br/>
[2022-07-25]修复http无法连接，而https和缺省的时候正常<br/>
[2022-06-30]新增enablePathStyleAccess、signerOverride后台配置<br/>
[2022-06-13]第三方组件升级<br/>

65
src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
Unescape View File

@ -6,33 +6,31 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit;
import com.fr.config.utils.RandomUtils;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.log.FineLoggerFactory;
import com.fr.stable.Filter;
import com.fr.workspace.resource.ResourceIOException;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
@ -45,10 +43,13 @@ public class S3ResourceRepository extends BaseResourceRepository {
private static final String DELIMITER = "/";
/**
* 1.5G
* 1G
*/
private static final int LENGTH_LIMIT = 1610612736;
private static final String LINK_SYMBOL = "_";
private static final int LENGTH_LIMIT = 1024 * 1024 * 1024;
/**
* 10M一个分片大小
*/
private static final int PART_SIZE = 10 * 1024 * 1024;
private final AmazonS3 readClient;
private final AmazonS3 writeClient;
@ -160,26 +161,30 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
// 大于1.5G的,就自己算一下长度
if (inputStream.available() > LENGTH_LIMIT) {
final long l1 = System.currentTimeMillis();
final File file = new File(l1 + LINK_SYMBOL + RandomUtils.randomId(5));
Files.deleteIfExists(file.toPath());
file.createNewFile();
try (final BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(file));) {
final long l = inputStream.transferTo(bufferedOutputStream);
bufferedOutputStream.flush();
metadata.setContentLength(l);
TransferManager tm = TransferManagerBuilder.standard()
.withS3Client(this.writeClient)
.build();
// TransferManager processes all transfers asynchronously,
// so this call returns immediately.
Upload upload = tm.upload(this.bucket, path, file);
// Optionally, wait for the upload to finish before continuing.
upload.waitForCompletion();
} finally {
Files.deleteIfExists(file.toPath());
List<PartETag> partETags = new ArrayList<>();
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = this.writeClient.initiateMultipartUpload(initRequest);
final String uploadId = initResponse.getUploadId();
byte[] buffer = new byte[PART_SIZE];
int bytesRead;
int partNumber = 1;
while ((bytesRead = inputStream.read(buffer)) > 0) {
// Create the request to upload a part.
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
.withPartSize(bytesRead);
final UploadPartResult uploadPartResult = this.writeClient.uploadPart(uploadRequest);
partETags.add(uploadPartResult.getPartETag());
partNumber++;
}
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path,
initResponse.getUploadId(), partETags);
this.writeClient.completeMultipartUpload(compRequest);
} else {
metadata.setContentLength(inputStream.available());
this.writeClient.putObject(this.bucket, path, inputStream, metadata);

Write Preview
Loading…
Cancel
Save