fanruan
/
plugin-repository-s3
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Pull request #409: 11.0-插件紧急发布 

Merge in PG/plugin-repository-s3 from release/11.0 to persist/11.0

* commit '3a89ec51f6b8ee080624f52653defe879252ec1c':
  无jira任务,S3默认配置设置为false
  无jira任务 fix: 提升插件版本
  JSY-28624 fix: 优化下分片上传逻辑
  REPORT-102146 fix: isDirectory方法某些场景判断不正确
  打包
  REPORT-99625 fix:enablePathStyleAccess默认false
  REPORT-99625 fix:enablePathStyleAccess默认false
persist/11.0
Icey.Zhang-张洁 1 year ago
parent
923dd4261a 3a89ec51f6
commit
f5a08831a9
4 changed files with 68 additions and 77 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      front/bundle.js
  2. 3
      plugin.xml
  3. 138
      src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
  4. 2
      src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js

2
front/bundle.js
Unescape View File

@ -164,7 +164,7 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: "PathStyleAccess",
value: String(this.model.enablePathStyleAccess),
value: () => this.model.enablePathStyleAccess,
ref: function (_ref) {
self.enablePathStyleAccessRow = _ref;
},

3
plugin.xml
Unescape View File

@ -5,12 +5,13 @@
<main-package>com.fanruan.fs</main-package>
<active>yes</active>
<hidden>no</hidden>
<version>1.3.9</version>
<version>1.4.0</version>
<env-version>11.0~11.0</env-version>
<jartime>2023-03-14</jartime>
<vendor>richie</vendor>
<description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
<change-notes><![CDATA[
[2023-08-08]分片上传逻辑优化，模板保存问题修复。 <br/>
[2023-07-24]支持大文件分片上传。 <br/>
[2023-06-30]修复默认配置获取错误的问题，过滤有问题的路径。 <br/>
[2023-03-28]第三方组件升级。 <br/>

138
src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
Unescape View File

@ -27,7 +27,6 @@ import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.io.utils.ResourceIOUtils;
import com.fr.log.FineLoggerFactory;
import com.fr.stable.Filter;
import com.fr.stable.StringUtils;
import com.fr.third.org.apache.commons.io.output.NullOutputStream;
@ -52,7 +51,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
private static final int PART_SIZE = 5 * 1024 * 1024;
private static final int MULTIPART_UPLOAD_LIMIT = 4 * PART_SIZE;
private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
private static final String DELIMITER = "/";
@ -155,75 +154,69 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public void write(String path, byte[] data) {
ObjectMetadata metadata;
try {
metadata = s3.getObjectMetadata(bucket, path);
} catch (Exception e) {
metadata = new ObjectMetadata();
String mimeType = URLConnection.guessContentTypeFromName(path);
if (mimeType != null) {
metadata.setContentType(mimeType);
int length = data.length;
if (length > MULTIPART_UPLOAD_LIMIT) {
multipartUpload(path, new ByteArrayInputStream(data));
} else {
ObjectMetadata metadata;
try {
metadata = s3.getObjectMetadata(bucket, path);
} catch (Exception e) {
metadata = new ObjectMetadata();
String mimeType = URLConnection.guessContentTypeFromName(path);
if (mimeType != null) {
metadata.setContentType(mimeType);
}
}
if (metadata != null) {
metadata.setContentLength(length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
if (metadata != null) {
metadata.setContentLength(data.length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
@Override
public void write(String path, InputStream inputStream) throws ResourceIOException {
long dataLength = 0;
multipartUpload(path, inputStream);
}
private void multipartUpload(String path, InputStream inputStream) {
try {
dataLength = inputStream.available();
} catch (IOException e) {
FineLoggerFactory.getLogger().error(e.getMessage(), e);
}
//超过一定大小才使用分片上传，小文件来说，网络传输时间可能较短，且上传失败的风险相对较低。
//在网络稳定的情况下，使用分片上传可能没有太多的优势，反而增加了额外开销和复杂性
if (dataLength > MULTIPART_UPLOAD_LIMIT) {
try {
// Step 1: 初始化分片上传
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
// Step 2: 分片上传文件
List<PartETag> partETags = new ArrayList<>();
long position = 0;
for (int partNumber = 1; position < dataLength; partNumber++) {
// 最后一个分片可能小于5MB
long partSizeBytes = Math.min(PART_SIZE, dataLength - position);
byte[] bytes = new byte[(int) partSizeBytes];
inputStream.read(bytes);
// 创建上传请求
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(bytes))
.withPartSize(partSizeBytes);
// 上传分片
UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
position += partSizeBytes;
}
// Step 1: 初始化分片上传
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
// Step 2: 分片上传文件
List<PartETag> partETags = new ArrayList<>();
byte[] buffer = new byte[PART_SIZE];
int bytesRead;
int partNumber = 1;
while ((bytesRead = inputStream.read(buffer)) > 0) {
// 创建上传请求
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
.withPartSize(bytesRead);
// Step 3: 完成分片上传
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
s3.completeMultipartUpload(compRequest);
} catch (IOException e) {
throw new ResourceIOException(e);
} finally {
ResourceIOUtils.close(inputStream);
// 上传分片
UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
partNumber++;
}
} else {
super.write(path, inputStream);
// Step 3: 完成分片上传
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
s3.completeMultipartUpload(compRequest);
} catch (IOException e) {
throw new ResourceIOException(e);
} finally {
ResourceIOUtils.close(inputStream);
}
}
@ -399,19 +392,16 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean isDirectory(String path) {
if (path.endsWith(DELIMITER) && exist(path)) {
return true;
}
ObjectListing listing = s3.listObjects(bucket, path);
if (listing.getObjectSummaries().isEmpty()) {
return false;
}
if (listing.getObjectSummaries().size() > 1) {
return true;
if (path.endsWith(DELIMITER)) {
return exist(path);
} else {
S3ObjectSummary summary = listing.getObjectSummaries().get(0);
return !StringKit.equals(listing.getPrefix(), summary.getKey());
ObjectListing listing = s3.listObjects(bucket, path);
List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
if (objectSummaries.isEmpty()) {
return false;
}
String dirFormat = path + DELIMITER;
return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
}
}

2
src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
View File

File diff suppressed because one or more lines are too long
Write Preview
Loading…
Cancel
Save