diff --git a/front/bundle.js b/front/bundle.js
index e042179..c3fb23a 100644
--- a/front/bundle.js
+++ b/front/bundle.js
@@ -164,7 +164,7 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: "PathStyleAccess",
- value: String(this.model.enablePathStyleAccess),
+ value: () => this.model.enablePathStyleAccess,
ref: function (_ref) {
self.enablePathStyleAccessRow = _ref;
},
diff --git a/plugin.xml b/plugin.xml
index fe32057..0466216 100644
--- a/plugin.xml
+++ b/plugin.xml
@@ -5,12 +5,13 @@
com.fanruan.fs
yes
no
- 1.3.9
+ 1.4.0
11.0~11.0
2023-03-14
richie
[2023-07-24]支持大文件分片上传。
[2023-06-30]修复默认配置获取错误的问题，过滤有问题的路径。
[2023-03-28]第三方组件升级。
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
index e9aa347..3e9455e 100644
--- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
+++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
@@ -27,7 +27,6 @@ import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.io.utils.ResourceIOUtils;
-import com.fr.log.FineLoggerFactory;
import com.fr.stable.Filter;
import com.fr.stable.StringUtils;
import com.fr.third.org.apache.commons.io.output.NullOutputStream;
@@ -52,7 +51,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
private static final int PART_SIZE = 5 * 1024 * 1024;
- private static final int MULTIPART_UPLOAD_LIMIT = 4 * PART_SIZE;
+ private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
private static final String DELIMITER = "/";
@@ -155,75 +154,69 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public void write(String path, byte[] data) {
- ObjectMetadata metadata;
- try {
- metadata = s3.getObjectMetadata(bucket, path);
- } catch (Exception e) {
- metadata = new ObjectMetadata();
- String mimeType = URLConnection.guessContentTypeFromName(path);
- if (mimeType != null) {
- metadata.setContentType(mimeType);
+ int length = data.length;
+ if (length > MULTIPART_UPLOAD_LIMIT) {
+ multipartUpload(path, new ByteArrayInputStream(data));
+ } else {
+ ObjectMetadata metadata;
+ try {
+ metadata = s3.getObjectMetadata(bucket, path);
+ } catch (Exception e) {
+ metadata = new ObjectMetadata();
+ String mimeType = URLConnection.guessContentTypeFromName(path);
+ if (mimeType != null) {
+ metadata.setContentType(mimeType);
+ }
}
+ if (metadata != null) {
+ metadata.setContentLength(length);
+ }
+ s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
- if (metadata != null) {
- metadata.setContentLength(data.length);
- }
- s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
@Override
public void write(String path, InputStream inputStream) throws ResourceIOException {
- long dataLength = 0;
+ multipartUpload(path, inputStream);
+ }
+
+ private void multipartUpload(String path, InputStream inputStream) {
try {
- dataLength = inputStream.available();
- } catch (IOException e) {
- FineLoggerFactory.getLogger().error(e.getMessage(), e);
- }
- //超过一定大小才使用分片上传，小文件来说，网络传输时间可能较短，且上传失败的风险相对较低。
- //在网络稳定的情况下，使用分片上传可能没有太多的优势，反而增加了额外开销和复杂性
- if (dataLength > MULTIPART_UPLOAD_LIMIT) {
- try {
- // Step 1: 初始化分片上传
- InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
- InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
- String uploadId = initResponse.getUploadId();
-
- // Step 2: 分片上传文件
- List partETags = new ArrayList<>();
- long position = 0;
-
- for (int partNumber = 1; position < dataLength; partNumber++) {
- // 最后一个分片可能小于5MB
- long partSizeBytes = Math.min(PART_SIZE, dataLength - position);
- byte[] bytes = new byte[(int) partSizeBytes];
- inputStream.read(bytes);
-
- // 创建上传请求
- UploadPartRequest uploadRequest = new UploadPartRequest()
- .withBucketName(bucket)
- .withKey(path)
- .withUploadId(uploadId)
- .withPartNumber(partNumber)
- .withInputStream(new ByteArrayInputStream(bytes))
- .withPartSize(partSizeBytes);
-
- // 上传分片
- UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
- partETags.add(uploadResult.getPartETag());
-
- position += partSizeBytes;
- }
+ // Step 1: 初始化分片上传
+ InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
+ InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
+ String uploadId = initResponse.getUploadId();
+
+ // Step 2: 分片上传文件
+ List partETags = new ArrayList<>();
+ byte[] buffer = new byte[PART_SIZE];
+ int bytesRead;
+ int partNumber = 1;
+
+ while ((bytesRead = inputStream.read(buffer)) > 0) {
+ // 创建上传请求
+ UploadPartRequest uploadRequest = new UploadPartRequest()
+ .withBucketName(bucket)
+ .withKey(path)
+ .withUploadId(uploadId)
+ .withPartNumber(partNumber)
+ .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
+ .withPartSize(bytesRead);
- // Step 3: 完成分片上传
- CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
- s3.completeMultipartUpload(compRequest);
- } catch (IOException e) {
- throw new ResourceIOException(e);
- } finally {
- ResourceIOUtils.close(inputStream);
+ // 上传分片
+ UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
+ partETags.add(uploadResult.getPartETag());
+
+ partNumber++;
}
- } else {
- super.write(path, inputStream);
+
+ // Step 3: 完成分片上传
+ CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
+ s3.completeMultipartUpload(compRequest);
+ } catch (IOException e) {
+ throw new ResourceIOException(e);
+ } finally {
+ ResourceIOUtils.close(inputStream);
}
}
@@ -399,19 +392,16 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean isDirectory(String path) {
-
- if (path.endsWith(DELIMITER) && exist(path)) {
- return true;
- }
- ObjectListing listing = s3.listObjects(bucket, path);
- if (listing.getObjectSummaries().isEmpty()) {
- return false;
- }
- if (listing.getObjectSummaries().size() > 1) {
- return true;
+ if (path.endsWith(DELIMITER)) {
+ return exist(path);
} else {
- S3ObjectSummary summary = listing.getObjectSummaries().get(0);
- return !StringKit.equals(listing.getPrefix(), summary.getKey());
+ ObjectListing listing = s3.listObjects(bucket, path);
+ List objectSummaries = listing.getObjectSummaries();
+ if (objectSummaries.isEmpty()) {
+ return false;
+ }
+ String dirFormat = path + DELIMITER;
+ return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
}
}
diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
index 5745993..7cf84df 100644
--- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
+++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
@@ -1 +1 @@
-BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:String(Boolean(e.enablePathStyleAccess)),signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}();
\ No newline at end of file
+BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:()=>this.model.enablePathStyleAccess,ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:String(Boolean(e.enablePathStyleAccess)),signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}();
\ No newline at end of file