From 8ad56d3eb3c7901d9a0ce5c32a41662819c7f1b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aeolus=2EZhang-=E5=BC=A0=E6=95=AC=E5=B3=A5?= Date: Tue, 6 Jun 2023 13:54:13 +0800 Subject: [PATCH 01/11] =?UTF-8?q?REPORT-97243=20fix:=E5=80=BC=E6=98=BE?= =?UTF-8?q?=E7=A4=BAundefined?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/bundle.js | 2 +- .../resources/com/fanruan/fs/s3/repository/web/js/bundle.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/front/bundle.js b/front/bundle.js index 600aa2e..858162f 100644 --- a/front/bundle.js +++ b/front/bundle.js @@ -262,7 +262,7 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) { bucket: val.bucket, workRoot: val.workRoot, isOpen: false, - enablePathStyleAccess: val.enablePathStyleAccess, + enablePathStyleAccess: val.enablePathStyleAccess ?? '', signerOverride: val.signerOverride, }; }, diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js index 9bb6c61..bfd07ae 100644 --- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js +++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js @@ -1 +1 @@ -BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return console.log(this.model.enablePathStyleAccess),{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return console.log(this.options.value),{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:e.enablePathStyleAccess,signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file +BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return console.log(this.model.enablePathStyleAccess),{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return console.log(this.options.value),{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:e.enablePathStyleAccess??"",signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file From 983bb3bee4c2321548dca632d713240a6e12d78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aeolus=2EZhang-=E5=BC=A0=E6=95=AC=E5=B3=A5?= Date: Tue, 6 Jun 2023 14:02:39 +0800 Subject: [PATCH 02/11] console --- front/bundle.js | 1 - 1 file changed, 1 deletion(-) diff --git a/front/bundle.js b/front/bundle.js index 858162f..68911e6 100644 --- a/front/bundle.js +++ b/front/bundle.js @@ -252,7 +252,6 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) { state: function () { var val = this.options.value; - console.log(this.options.value); return { endPoint: val.endPoint, From 729dadf90d2e5445623d7c4d236f206d30600447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aeolus=2EZhang-=E5=BC=A0=E6=95=AC=E5=B3=A5?= Date: Thu, 8 Jun 2023 10:18:59 +0800 Subject: [PATCH 03/11] =?UTF-8?q?enablePathStyleAccess=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/bundle.js | 3 +-- .../resources/com/fanruan/fs/s3/repository/web/js/bundle.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/front/bundle.js b/front/bundle.js index 68911e6..c0ca26e 100644 --- a/front/bundle.js +++ b/front/bundle.js @@ -28,7 +28,6 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) { render: function () { var self = this, o = this.options; - console.log(this.model.enablePathStyleAccess); return { type: "bi.vertical", @@ -261,7 +260,7 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) { bucket: val.bucket, workRoot: val.workRoot, isOpen: false, - enablePathStyleAccess: val.enablePathStyleAccess ?? '', + enablePathStyleAccess: String(val.enablePathStyleAccess) ?? '', signerOverride: val.signerOverride, }; }, diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js index bfd07ae..057554d 100644 --- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js +++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js @@ -1 +1 @@ -BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return console.log(this.model.enablePathStyleAccess),{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return console.log(this.options.value),{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:e.enablePathStyleAccess??"",signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file +BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:String(e.enablePathStyleAccess)??"",signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file From d5e6992ddad73125fe6df1b43f2db752a64ab00c Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:13 +0800 Subject: [PATCH 04/11] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=B1=A0=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fanruan/fs/s3/repository/core/S3Config.java | 15 +++++++++++++++ .../s3/repository/core/S3ResourceRepository.java | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java index 5821864..976a9b8 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java @@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig { @Identifier("signerOverride") private Conf signerOverride = HolderKit.simple(StringUtils.EMPTY); + @Identifier("maxConnections") + private Conf maxConnections = HolderKit.simple(200); + @GetConfig("endPoint") public String getEndPoint() { return endPoint.get(); @@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig { this.signerOverride.set(signerOverride); } + @GetConfig("maxConnections") + public int getMaxConnections() { + return maxConnections.get(); + } + + @SetConfig("maxConnections") + public void setMaxConnections(int maxConnections) { + this.maxConnections.set(maxConnections); + } + @Override public void update(String key) { super.update(key); @@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig { this.setBucket(newConfig.getBucket()); this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess()); this.setSignerOverride(newConfig.getSignerOverride()); + this.setMaxConnections(newConfig.getMaxConnections()); } } @@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig { cloned.bucket = (Conf) bucket.clone(); cloned.enablePathStyleAccess = (Conf) enablePathStyleAccess.clone(); cloned.signerOverride = (Conf) signerOverride.clone(); + cloned.maxConnections = (Conf) maxConnections.clone(); return cloned; } } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 4cb90ec..bfd7c47 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -15,6 +15,7 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; @@ -57,6 +58,8 @@ public class S3ResourceRepository extends BaseResourceRepository { amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); } ClientConfiguration clientConfiguration = new ClientConfiguration(); + clientConfiguration.setMaxConnections(config.getMaxConnections()); + LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections()); if (StringUtils.isNotEmpty(config.getSignerOverride())) { clientConfiguration.setSignerOverride(config.getSignerOverride()); } @@ -128,6 +131,8 @@ public class S3ResourceRepository extends BaseResourceRepository { try { return s3.getObject(request).getObjectContent(); } catch (Exception e) { + LogKit.error("[S3] Failed to read file {}", filePath); + LogKit.error(e.getMessage(), e); return new ByteArrayInputStream(new byte[0]); } } @@ -341,6 +346,8 @@ public class S3ResourceRepository extends BaseResourceRepository { S3Object s3Object = s3.getObject(bucket, path); if (s3Object != null) { try { + //s3Object要全部读完，否则会有警告 + IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream()); return s3Object.getObjectMetadata().getLastModified().getTime(); } finally { s3Object.close(); From 974c7e2b4265a55ec62574f7ef10942a3680ea17 Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:36 +0800 Subject: [PATCH 05/11] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=B1=A0=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fanruan/fs/s3/repository/core/S3ResourceRepository.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index bfd7c47..0f7b77a 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -22,6 +22,7 @@ import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; import com.fr.stable.Filter; import com.fr.stable.StringUtils; +import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream; From 2c8bab646c5dd77ff15341b0df35a883144ae450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aeolus=2EZhang-=E5=BC=A0=E6=95=AC=E5=B3=A5?= Date: Wed, 14 Jun 2023 11:19:29 +0800 Subject: [PATCH 06/11] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/bundle.js | 2 +- .../resources/com/fanruan/fs/s3/repository/web/js/bundle.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/front/bundle.js b/front/bundle.js index c0ca26e..96f371a 100644 --- a/front/bundle.js +++ b/front/bundle.js @@ -260,7 +260,7 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) { bucket: val.bucket, workRoot: val.workRoot, isOpen: false, - enablePathStyleAccess: String(val.enablePathStyleAccess) ?? '', + enablePathStyleAccess: val.enablePathStyleAccess ? String(val.enablePathStyleAccess) : '', signerOverride: val.signerOverride, }; }, diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js index 057554d..1cc22d0 100644 --- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js +++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js @@ -1 +1 @@ -BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:String(e.enablePathStyleAccess)??"",signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file +BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:e.enablePathStyleAccess?String(e.enablePathStyleAccess):"",signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file From 2bd592e352bd679639d561353fdd61dfe0576a58 Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 15 Jun 2023 09:58:41 +0800 Subject: [PATCH 07/11] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20fix:=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=BF=9E=E6=8E=A5=E5=90=8E=E9=87=8A=E6=94=BE?= =?UTF-8?q?=E8=B5=84=E6=BA=90;list=E6=8E=A5=E5=8F=A3=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E8=87=AA=E8=BA=AB=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3RepositoryFactory.java | 7 ++++++- .../repository/core/S3ResourceRepository.java | 20 ++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java index 2aecad2..5de506c 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java @@ -40,6 +40,7 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { @Override public boolean verifyConfig(S3Config config) { + AmazonS3 s3 = null; try { BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword()); AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() @@ -57,11 +58,15 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { clientConfiguration.setProtocol(Protocol.HTTP); } amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); - AmazonS3 s3 = amazonS3ClientBuilder.build(); + s3 = amazonS3ClientBuilder.build(); s3.listObjects(config.getBucket()); } catch (Exception e) { LogKit.error(e.getMessage(), e); return false; + } finally { + if (s3 != null) { + s3.shutdown(); + } } return true; } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 0f7b77a..d771bd2 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -99,15 +99,15 @@ public class S3ResourceRepository extends BaseResourceRepository { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) .withPrefix(dir).withDelimiter(DELIMITER); ObjectListing objectListing = s3.listObjects(listObjectsRequest); - collectFileEntry(result, objectListing); + collectFileEntry(dir, result, objectListing); while (objectListing.isTruncated()) { objectListing = s3.listNextBatchOfObjects(objectListing); - collectFileEntry(result, objectListing); + collectFileEntry(dir, result, objectListing); } return result.toArray(new FineFileEntry[0]); } - private void collectFileEntry(List result, ObjectListing objectListing) { + private void collectFileEntry(String dir, List result, ObjectListing objectListing) { for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { String key = summary.getKey(); if (!key.endsWith(DELIMITER)) { @@ -115,9 +115,11 @@ public class S3ResourceRepository extends BaseResourceRepository { } } for (String prefix : objectListing.getCommonPrefixes()) { - FineFileEntry entry = new FineFileEntry(prefix); - entry.setDirectory(true); - result.add(entry); + if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) { + FineFileEntry entry = new FineFileEntry(prefix); + entry.setDirectory(true); + result.add(entry); + } } } @@ -319,7 +321,11 @@ public class S3ResourceRepository extends BaseResourceRepository { result.add(key.substring(key.lastIndexOf(DELIMITER) + 1)); } for (String prefix : objectListing.getCommonPrefixes()) { - result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1)); + if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) { + String[] arr = prefix.split(DELIMITER); + String name = arr[arr.length - 1] + DELIMITER; + result.add(name); + } } } From 9c5b9ffcc0507c3ad95677b52bd785a3a077b990 Mon Sep 17 00:00:00 2001 From: Afly Date: Fri, 30 Jun 2023 17:19:41 +0800 Subject: [PATCH 08/11] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20chore:?= =?UTF-8?q?=20=E6=8F=90=E5=8D=87=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin.xml b/plugin.xml index 99d08b2..5e58232 100644 --- a/plugin.xml +++ b/plugin.xml @@ -5,12 +5,13 @@ com.fanruan.fs yes no - 1.3.8 + 1.3.9 11.0~11.0 2023-03-14 richie [2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。
[2022-09-22]第三方组件升级。
From e6d47da394cd5fa38d188733c611318b836ac1ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aeolus=2EZhang-=E5=BC=A0=E6=95=AC=E5=B3=A5?= Date: Mon, 3 Jul 2023 14:58:00 +0800 Subject: [PATCH 09/11] =?UTF-8?q?REPORT-99625=20fix:enablePathStyleAccess?= =?UTF-8?q?=E9=BB=98=E8=AE=A4false?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/bundle.js | 2 +- .../resources/com/fanruan/fs/s3/repository/web/js/bundle.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/front/bundle.js b/front/bundle.js index 96f371a..e042179 100644 --- a/front/bundle.js +++ b/front/bundle.js @@ -260,7 +260,7 @@ BI.config("dec.constant.intelligence.cluster.file.server", function (items) { bucket: val.bucket, workRoot: val.workRoot, isOpen: false, - enablePathStyleAccess: val.enablePathStyleAccess ? String(val.enablePathStyleAccess) : '', + enablePathStyleAccess: String(Boolean(val.enablePathStyleAccess)), signerOverride: val.signerOverride, }; }, diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js index 1cc22d0..5745993 100644 --- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js +++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js @@ -1 +1 @@ -BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:e.enablePathStyleAccess?String(e.enablePathStyleAccess):"",signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file +BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:String(this.model.enablePathStyleAccess),ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCompleteCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:String(Boolean(e.enablePathStyleAccess)),signerOverride:e.signerOverride}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file From 047402bc02e3a9461cc9cb4af27c9766e1dee15d Mon Sep 17 00:00:00 2001 From: Afly Date: Mon, 24 Jul 2023 20:24:42 +0800 Subject: [PATCH 10/11] =?UTF-8?q?REPORT-100817=20fix:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=AF=B9=E4=BA=8E=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3ResourceRepository.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index d771bd2..e9aa347 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -7,25 +7,34 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; +import com.fr.io.utils.ResourceIOUtils; +import com.fr.log.FineLoggerFactory; import com.fr.stable.Filter; import com.fr.stable.StringUtils; import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; @@ -41,6 +50,10 @@ public class S3ResourceRepository extends BaseResourceRepository { private static final int PAGE_SIZE = 1000; + private static final int PART_SIZE = 5 * 1024 * 1024; + + private static final int MULTIPART_UPLOAD_LIMIT = 4 * PART_SIZE; + private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -158,6 +171,61 @@ public class S3ResourceRepository extends BaseResourceRepository { s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); } + @Override + public void write(String path, InputStream inputStream) throws ResourceIOException { + long dataLength = 0; + try { + dataLength = inputStream.available(); + } catch (IOException e) { + FineLoggerFactory.getLogger().error(e.getMessage(), e); + } + //超过一定大小才使用分片上传，小文件来说，网络传输时间可能较短，且上传失败的风险相对较低。 + //在网络稳定的情况下，使用分片上传可能没有太多的优势，反而增加了额外开销和复杂性 + if (dataLength > MULTIPART_UPLOAD_LIMIT) { + try { + // Step 1: 初始化分片上传 + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path); + InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + + // Step 2: 分片上传文件 + List partETags = new ArrayList<>(); + long position = 0; + + for (int partNumber = 1; position < dataLength; partNumber++) { + // 最后一个分片可能小于5MB + long partSizeBytes = Math.min(PART_SIZE, dataLength - position); + byte[] bytes = new byte[(int) partSizeBytes]; + inputStream.read(bytes); + + // 创建上传请求 + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucket) + .withKey(path) + .withUploadId(uploadId) + .withPartNumber(partNumber) + .withInputStream(new ByteArrayInputStream(bytes)) + .withPartSize(partSizeBytes); + + // 上传分片 + UploadPartResult uploadResult = s3.uploadPart(uploadRequest); + partETags.add(uploadResult.getPartETag()); + + position += partSizeBytes; + } + + // Step 3: 完成分片上传 + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags); + s3.completeMultipartUpload(compRequest); + } catch (IOException e) { + throw new ResourceIOException(e); + } finally { + ResourceIOUtils.close(inputStream); + } + } else { + super.write(path, inputStream); + } + } @Override public boolean createFile(String path) { From d4e744b798952cf72cbfddd5f86234688558e1c4 Mon Sep 17 00:00:00 2001 From: Afly Date: Mon, 24 Jul 2023 20:27:35 +0800 Subject: [PATCH 11/11] =?UTF-8?q?REPORT-100817=20fix:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=AF=B9=E4=BA=8E=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin.xml b/plugin.xml index 5e58232..fe32057 100644 --- a/plugin.xml +++ b/plugin.xml @@ -11,6 +11,7 @@ richie [2023-06-30]修复默认配置获取错误的问题，过滤有问题的路径。
[2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。