fanruan
/
plugin-repository-s3
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Pull request #401: REPORT-102146 && JSY-28624 fix: isDirectory方法某些场景判断不正确；优化下分片上传逻辑 

Merge in PG/plugin-repository-s3 from ~AFLY/plugin-repository-s3:release/11.0 to release/11.0

* commit '8051a92c88db7c113c585ddb55436d9b0f5d1098':
  JSY-28624 fix: 优化下分片上传逻辑
  REPORT-102146 fix: isDirectory方法某些场景判断不正确
release/11.0
Afly-储泓飞 1 year ago
parent
b502148359 8051a92c88
commit
cfc791698d
1 changed files with 64 additions and 74 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 138
      src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

138
src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
Unescape View File

@ -27,7 +27,6 @@ import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.io.utils.ResourceIOUtils;
import com.fr.log.FineLoggerFactory;
import com.fr.stable.Filter;
import com.fr.stable.StringUtils;
import com.fr.third.org.apache.commons.io.output.NullOutputStream;
@ -52,7 +51,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
private static final int PART_SIZE = 5 * 1024 * 1024;
private static final int MULTIPART_UPLOAD_LIMIT = 4 * PART_SIZE;
private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
private static final String DELIMITER = "/";
@ -155,75 +154,69 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public void write(String path, byte[] data) {
ObjectMetadata metadata;
try {
metadata = s3.getObjectMetadata(bucket, path);
} catch (Exception e) {
metadata = new ObjectMetadata();
String mimeType = URLConnection.guessContentTypeFromName(path);
if (mimeType != null) {
metadata.setContentType(mimeType);
int length = data.length;
if (length > MULTIPART_UPLOAD_LIMIT) {
multipartUpload(path, new ByteArrayInputStream(data));
} else {
ObjectMetadata metadata;
try {
metadata = s3.getObjectMetadata(bucket, path);
} catch (Exception e) {
metadata = new ObjectMetadata();
String mimeType = URLConnection.guessContentTypeFromName(path);
if (mimeType != null) {
metadata.setContentType(mimeType);
}
}
if (metadata != null) {
metadata.setContentLength(length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
if (metadata != null) {
metadata.setContentLength(data.length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
@Override
public void write(String path, InputStream inputStream) throws ResourceIOException {
long dataLength = 0;
multipartUpload(path, inputStream);
}
private void multipartUpload(String path, InputStream inputStream) {
try {
dataLength = inputStream.available();
} catch (IOException e) {
FineLoggerFactory.getLogger().error(e.getMessage(), e);
}
//超过一定大小才使用分片上传，小文件来说，网络传输时间可能较短，且上传失败的风险相对较低。
//在网络稳定的情况下，使用分片上传可能没有太多的优势，反而增加了额外开销和复杂性
if (dataLength > MULTIPART_UPLOAD_LIMIT) {
try {
// Step 1: 初始化分片上传
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
// Step 2: 分片上传文件
List<PartETag> partETags = new ArrayList<>();
long position = 0;
for (int partNumber = 1; position < dataLength; partNumber++) {
// 最后一个分片可能小于5MB
long partSizeBytes = Math.min(PART_SIZE, dataLength - position);
byte[] bytes = new byte[(int) partSizeBytes];
inputStream.read(bytes);
// 创建上传请求
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(bytes))
.withPartSize(partSizeBytes);
// 上传分片
UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
position += partSizeBytes;
}
// Step 1: 初始化分片上传
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
// Step 2: 分片上传文件
List<PartETag> partETags = new ArrayList<>();
byte[] buffer = new byte[PART_SIZE];
int bytesRead;
int partNumber = 1;
while ((bytesRead = inputStream.read(buffer)) > 0) {
// 创建上传请求
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
.withPartSize(bytesRead);
// Step 3: 完成分片上传
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
s3.completeMultipartUpload(compRequest);
} catch (IOException e) {
throw new ResourceIOException(e);
} finally {
ResourceIOUtils.close(inputStream);
// 上传分片
UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
partNumber++;
}
} else {
super.write(path, inputStream);
// Step 3: 完成分片上传
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
s3.completeMultipartUpload(compRequest);
} catch (IOException e) {
throw new ResourceIOException(e);
} finally {
ResourceIOUtils.close(inputStream);
}
}
@ -399,19 +392,16 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean isDirectory(String path) {
if (path.endsWith(DELIMITER) && exist(path)) {
return true;
}
ObjectListing listing = s3.listObjects(bucket, path);
if (listing.getObjectSummaries().isEmpty()) {
return false;
}
if (listing.getObjectSummaries().size() > 1) {
return true;
if (path.endsWith(DELIMITER)) {
return exist(path);
} else {
S3ObjectSummary summary = listing.getObjectSummaries().get(0);
return !StringKit.equals(listing.getPrefix(), summary.getKey());
ObjectListing listing = s3.listObjects(bucket, path);
List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
if (objectSummaries.isEmpty()) {
return false;
}
String dirFormat = path + DELIMITER;
return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
}
}

Write Preview
Loading…
Cancel
Save