Browse Source

Pull request #463: 10.0.19.34版本插件代码合并

Merge in PG/plugin-repository-s3 from release/10.0 to persist/10.0

* commit 'aecbffaa354e8f7256bfe4ae111f06eedb756f3b':
  无jira,回退错误提升的jartime
  无jira任务 fix: 提升插件版本
  JSY-28624 fix: 优化下分片上传逻辑
  REPORT-102146 fix: isDirectory方法某些场景判断不正确
  REPORT-100817 fix: s3插件对于分片上传的支持
  REPORT-99648 fix:s3 fineui兼容问题
  无jira任务 chore: 提升版本
  无jira任务 fix: 测试连接后释放资源;list接口过滤自身目录
  REPORT-97765 feat: s3插件支持配置连接池大小
  REPORT-97765 feat: s3插件支持配置连接池大小
  REPORT-96529 fix: list接口数量限制;删除接口性能优化
  无jira任务提升jartime和版本号
  REPORT-95454 bugfix:http配置根据endpoint进行设置
persist/10.0
Icey.Zhang-张洁 1 year ago
parent
commit
3649bfc1ec
  1. 4
      .gitignore
  2. 318
      front/bundle.js
  3. 10
      front/gulpfile.js
  4. 13
      front/package.json
  5. 4
      plugin.xml
  6. 15
      src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java
  7. 18
      src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java
  8. 247
      src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
  9. 312
      src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js

4
.gitignore vendored

@ -7,4 +7,6 @@ target/
build build
local.properties local.properties
classes/ classes/
transform-classes/ transform-classes/
node_modules
yarn.lock

318
front/bundle.js

@ -0,0 +1,318 @@
BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
items.push({
value: "S3", // 地址栏显示的hash值
id: "decision-intelligence-cluster-file-s3", // id
text: "S3", // 文字
cardType: "dec.intelligence.cluster.file.s3",
workRoot: false,
});
return items;
});
!(function () {
var LABEL_WIDTH = 107, EDITOR_WIDTH = 393;
var S3 = BI.inherit(BI.Widget, {
props: {
baseCls: "dec-cluster-ftp",
value: {},
},
_store: function () {
return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", {
value: this.options.value,
});
},
watch: {
isOpen: function (v) {
this.toggle.setVisible(v);
}
},
render: function () {
var self = this, o = this.options;
return {
type: "bi.vertical",
tgap: 15,
items: [
{
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: BI.i18nText("Plugin-S3_End_Point"),
value: this.model.endPoint,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.endPointRow = _ref;
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setEndPoint(this.getValue());
},
}],
}, {
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: BI.i18nText("Plugin-S3_Region"),
value: this.model.region,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.regionRow = _ref;
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setRegion(this.getValue());
},
}],
}, {
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: BI.i18nText("Plugin-S3_Access_Key_Id"),
value: this.model.accessKeyId,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.portRow = _ref;
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setAccessKeyId(this.getValue());
},
}],
},
{
type: "dec.common.cipher.editor",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"),
text: BI.i18nText("Plugin-S3_Access_Key_Secret"),
value: this.model.password,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.passwordRow = _ref;
},
},
{
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Bucket"),
text: BI.i18nText("Plugin-S3_Bucket"),
value: this.model.bucket,
el: {
disabled: !o.editable,
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setBucket(this.getValue());
},
}],
},
{
type: "dec.label.editor.item",
el: {
disabled: !o.editable,
},
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Dec-Please_Input"),
text: BI.i18nText("Dec-Basic_Path"),
value: this.model.workRoot,
ref: function (_ref) {
self.filePathRow = _ref;
},
}, {
type: "bi.vertical_adapt",
items: [{
type: "bi.icon_change_button",
iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font",
ref: (_ref) => {
this.OtherConfigButton = _ref;
},
handler: () => {
this.store.setIsOpen(!this.model.isOpen);
this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
}
}, {
type: "bi.text_button",
text: BI.i18nText('Plugin-S3_Other_Config'),
handler: () => {
this.store.setIsOpen(!this.model.isOpen);
this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
}
}]
}, {
type: 'bi.vertical',
ref: (_ref) => {
this.toggle = _ref;
},
invisible: () => !this.model.isOpen,
items: [{
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: "PathStyleAccess",
value: this.model.enablePathStyleAccess,
ref: function (_ref) {
self.enablePathStyleAccessRow = _ref;
},
el: {
disabled: !o.editable,
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setEnablePathStyleAccess(this.getValue());
}
}]
}, {
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: "SignerOverride",
value: this.model.signerOverride,
el: {
disabled: !o.editable,
},
tgap: 15,
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setSignerOverride(this.getValue());
}
}]
}],
}
]
};
},
getValue: function () {
var enablePathStyleAccess = false;
if (this.model.enablePathStyleAccess === 'true') {
enablePathStyleAccess = true;
}
return {
endPoint: this.model.endPoint,
region: this.model.region,
accessKeyId: this.model.accessKeyId,
password: this.passwordRow.getCipher(),
bucket: this.model.bucket,
workRoot: this.filePathRow.getValue(),
enablePathStyleAccess,
signerOverride: this.model.signerOverride,
};
},
validation: function () {
var valid = true;
var path = this.filePathRow.getValue();
if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) {
this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text"));
valid = false;
}
if (BI.startWith(path, "/") || !BI.endWith(path, "/")) {
this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash"));
valid = false;
}
if (!BI.isKey(path)) {
this.filePathRow.showError(BI.i18nText("Dec-Error_Null"));
valid = false;
}
if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) {
this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip"));
valid = false;
}
return valid;
},
});
BI.shortcut("dec.intelligence.cluster.file.s3", S3);
}());
!(function () {
var Model = BI.inherit(Fix.Model, {
state: function () {
var val = this.options.value;
return {
endPoint: val.endPoint,
region: val.region,
accessKeyId: val.accessKeyId,
password: val.password,
bucket: val.bucket,
workRoot: val.workRoot,
isOpen: false,
enablePathStyleAccess: 'false',
signerOverride: "",
};
},
computed: {
encodingArray: function () {
return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) {
return {
value: v,
};
});
},
},
actions: {
setEndPoint: function (v) {
this.model.endPoint = v;
},
setRegion: function (v) {
this.model.region = v;
},
setAccessKeyId: function (v) {
this.model.accessKeyId = v;
},
setBucket: function (v) {
this.model.bucket = v;
},
setEnablePathStyleAccess: function (v) {
this.model.enablePathStyleAccess = v;
},
setSignerOverride: function (v) {
this.model.signerOverride = v;
},
setIsOpen: function (v) {
this.model.isOpen = v;
},
},
});
BI.model("dec.model.intelligence.cluster.file.s3", Model);
}());

10
front/gulpfile.js

@ -0,0 +1,10 @@
const gulp = require('gulp');
const uglify = require('gulp-uglify');
function build() {
return gulp.src('./bundle.js')
.pipe(uglify())
.pipe(gulp.dest('../src/main/resources/com/fanruan/fs/s3/repository/web/js'));
}
exports.default = build;

13
front/package.json

@ -0,0 +1,13 @@
{
"name": "front",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"scripts": {
"build": "gulp"
},
"dependencies": {
"gulp": "^4.0.2",
"gulp-uglify": "^3.0.2"
}
}

4
plugin.xml

@ -5,12 +5,14 @@
<main-package>com.fanruan.fs</main-package> <main-package>com.fanruan.fs</main-package>
<active>yes</active> <active>yes</active>
<hidden>no</hidden> <hidden>no</hidden>
<version>1.3.7</version> <version>1.4.0</version>
<env-version>10.0~10.0</env-version> <env-version>10.0~10.0</env-version>
<jartime>2021-03-11</jartime> <jartime>2021-03-11</jartime>
<vendor>richie</vendor> <vendor>richie</vendor>
<description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description> <description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
<change-notes><![CDATA[ <change-notes><![CDATA[
[2023-08-08]支持分片上传,模板保存问题修复。 <br/>
[2023-06-30]修复默认配置获取错误的问题,过滤有问题的路径。 <br/>
[2023-03-28]第三方组件升级。 <br/> [2023-03-28]第三方组件升级。 <br/>
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/> [2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/>
[2022-09-22]第三方组件升级。 <br/> [2022-09-22]第三方组件升级。 <br/>

15
src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java

@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig {
@Identifier("signerOverride") @Identifier("signerOverride")
private Conf<String> signerOverride = HolderKit.simple(StringUtils.EMPTY); private Conf<String> signerOverride = HolderKit.simple(StringUtils.EMPTY);
@Identifier("maxConnections")
private Conf<Integer> maxConnections = HolderKit.simple(200);
@GetConfig("endPoint") @GetConfig("endPoint")
public String getEndPoint() { public String getEndPoint() {
return endPoint.get(); return endPoint.get();
@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig {
this.signerOverride.set(signerOverride); this.signerOverride.set(signerOverride);
} }
@GetConfig("maxConnections")
public int getMaxConnections() {
return maxConnections.get();
}
@SetConfig("maxConnections")
public void setMaxConnections(int maxConnections) {
this.maxConnections.set(maxConnections);
}
@Override @Override
public void update(String key) { public void update(String key) {
super.update(key); super.update(key);
@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig {
this.setBucket(newConfig.getBucket()); this.setBucket(newConfig.getBucket());
this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess()); this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess());
this.setSignerOverride(newConfig.getSignerOverride()); this.setSignerOverride(newConfig.getSignerOverride());
this.setMaxConnections(newConfig.getMaxConnections());
} }
} }
@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig {
cloned.bucket = (Conf<String>) bucket.clone(); cloned.bucket = (Conf<String>) bucket.clone();
cloned.enablePathStyleAccess = (Conf<Boolean>) enablePathStyleAccess.clone(); cloned.enablePathStyleAccess = (Conf<Boolean>) enablePathStyleAccess.clone();
cloned.signerOverride = (Conf<String>) signerOverride.clone(); cloned.signerOverride = (Conf<String>) signerOverride.clone();
cloned.maxConnections = (Conf<Integer>) maxConnections.clone();
return cloned; return cloned;
} }
} }

18
src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java

@ -13,6 +13,8 @@ import com.fr.io.context.info.RepositoryProfile;
import com.fr.io.repository.ResourceRepository; import com.fr.io.repository.ResourceRepository;
import com.fr.stable.StringUtils; import com.fr.stable.StringUtils;
import static com.fanruan.fs.s3.repository.core.S3ResourceRepository.HTTP;
/** /**
* @author richie * @author richie
* @version 10.0 * @version 10.0
@ -38,25 +40,33 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory<S3Config> {
@Override @Override
public boolean verifyConfig(S3Config config) { public boolean verifyConfig(S3Config config) {
AmazonS3 s3 = null;
try { try {
BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword()); BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword());
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion())) .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion()))
.withCredentials(new AWSStaticCredentialsProvider(credentials)); .withCredentials(new AWSStaticCredentialsProvider(credentials)).disableChunkedEncoding();
if (config.isEnablePathStyleAccess()) { if (config.isEnablePathStyleAccess()) {
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
} }
ClientConfiguration clientConfiguration = new ClientConfiguration();
if (StringUtils.isNotEmpty(config.getSignerOverride())) { if (StringUtils.isNotEmpty(config.getSignerOverride())) {
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setSignerOverride(config.getSignerOverride()); clientConfiguration.setSignerOverride(config.getSignerOverride());
}
LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
if (config.getEndPoint().startsWith(HTTP)) {
clientConfiguration.setProtocol(Protocol.HTTP); clientConfiguration.setProtocol(Protocol.HTTP);
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
} }
AmazonS3 s3 = amazonS3ClientBuilder.build(); amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
s3 = amazonS3ClientBuilder.build();
s3.listObjects(config.getBucket()); s3.listObjects(config.getBucket());
} catch (Exception e) { } catch (Exception e) {
LogKit.error(e.getMessage(), e); LogKit.error(e.getMessage(), e);
return false; return false;
} finally {
if (s3 != null) {
s3.shutdown();
}
} }
return true; return true;
} }

247
src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@ -7,22 +7,33 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.amazonaws.util.IOUtils;
import com.fanruan.api.log.LogKit; import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit; import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository; import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.io.utils.ResourceIOUtils;
import com.fr.stable.Filter; import com.fr.stable.Filter;
import com.fr.stable.StringUtils; import com.fr.stable.StringUtils;
import com.fr.third.org.apache.commons.io.output.NullOutputStream;
import com.fr.workspace.resource.ResourceIOException; import com.fr.workspace.resource.ResourceIOException;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
@ -36,7 +47,14 @@ import java.util.List;
*/ */
public class S3ResourceRepository extends BaseResourceRepository { public class S3ResourceRepository extends BaseResourceRepository {
private static final int PAGE_SIZE = 1000;
private static final int PART_SIZE = 5 * 1024 * 1024;
private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
private static final String DELIMITER = "/"; private static final String DELIMITER = "/";
public static final String HTTP = "http:";
private final AmazonS3 s3; private final AmazonS3 s3;
private final String bucket; private final String bucket;
@ -52,10 +70,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
} }
ClientConfiguration clientConfiguration = new ClientConfiguration(); ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setMaxConnections(config.getMaxConnections());
LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections());
if (StringUtils.isNotEmpty(config.getSignerOverride())) { if (StringUtils.isNotEmpty(config.getSignerOverride())) {
clientConfiguration.setSignerOverride(config.getSignerOverride()); clientConfiguration.setSignerOverride(config.getSignerOverride());
} }
clientConfiguration.setProtocol(Protocol.HTTP); LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
if (config.getEndPoint().startsWith(HTTP)) {
clientConfiguration.setProtocol(Protocol.HTTP);
}
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
this.s3 = amazonS3ClientBuilder.build(); this.s3 = amazonS3ClientBuilder.build();
this.bucket = config.getBucket(); this.bucket = config.getBucket();
@ -87,15 +110,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER); .withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest); ObjectListing objectListing = s3.listObjects(listObjectsRequest);
collectFileEntry(result, objectListing); collectFileEntry(dir, result, objectListing);
while (objectListing.isTruncated()) { while (objectListing.isTruncated()) {
objectListing = s3.listNextBatchOfObjects(objectListing); objectListing = s3.listNextBatchOfObjects(objectListing);
collectFileEntry(result, objectListing); collectFileEntry(dir, result, objectListing);
} }
return result.toArray(new FineFileEntry[0]); return result.toArray(new FineFileEntry[0]);
} }
private void collectFileEntry(List<FineFileEntry> result, ObjectListing objectListing) { private void collectFileEntry(String dir, List<FineFileEntry> result, ObjectListing objectListing) {
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
String key = summary.getKey(); String key = summary.getKey();
if (!key.endsWith(DELIMITER)) { if (!key.endsWith(DELIMITER)) {
@ -103,9 +126,11 @@ public class S3ResourceRepository extends BaseResourceRepository {
} }
} }
for (String prefix : objectListing.getCommonPrefixes()) { for (String prefix : objectListing.getCommonPrefixes()) {
FineFileEntry entry = new FineFileEntry(prefix); if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) {
entry.setDirectory(true); FineFileEntry entry = new FineFileEntry(prefix);
result.add(entry); entry.setDirectory(true);
result.add(entry);
}
} }
} }
@ -120,28 +145,79 @@ public class S3ResourceRepository extends BaseResourceRepository {
try { try {
return s3.getObject(request).getObjectContent(); return s3.getObject(request).getObjectContent();
} catch (Exception e) { } catch (Exception e) {
LogKit.error("[S3] Failed to read file {}", filePath);
LogKit.error(e.getMessage(), e);
return new ByteArrayInputStream(new byte[0]); return new ByteArrayInputStream(new byte[0]);
} }
} }
@Override @Override
public void write(String path, byte[] data) { public void write(String path, byte[] data) {
ObjectMetadata metadata; int length = data.length;
try { if (length > MULTIPART_UPLOAD_LIMIT) {
metadata = s3.getObjectMetadata(bucket, path); multipartUpload(path, new ByteArrayInputStream(data));
} catch (Exception e) { } else {
metadata = new ObjectMetadata(); ObjectMetadata metadata;
String mimeType = URLConnection.guessContentTypeFromName(path); try {
if (mimeType != null) { metadata = s3.getObjectMetadata(bucket, path);
metadata.setContentType(mimeType); } catch (Exception e) {
metadata = new ObjectMetadata();
String mimeType = URLConnection.guessContentTypeFromName(path);
if (mimeType != null) {
metadata.setContentType(mimeType);
}
} }
if (metadata != null) {
metadata.setContentLength(length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
} }
if (metadata != null) {
metadata.setContentLength(data.length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
} }
@Override
public void write(String path, InputStream inputStream) throws ResourceIOException {
multipartUpload(path, inputStream);
}
private void multipartUpload(String path, InputStream inputStream) {
try {
// Step 1: 初始化分片上传
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
// Step 2: 分片上传文件
List<PartETag> partETags = new ArrayList<>();
byte[] buffer = new byte[PART_SIZE];
int bytesRead;
int partNumber = 1;
while ((bytesRead = inputStream.read(buffer)) > 0) {
// 创建上传请求
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
.withPartSize(bytesRead);
// 上传分片
UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
partNumber++;
}
// Step 3: 完成分片上传
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
s3.completeMultipartUpload(compRequest);
} catch (IOException e) {
throw new ResourceIOException(e);
} finally {
ResourceIOUtils.close(inputStream);
}
}
@Override @Override
public boolean createFile(String path) { public boolean createFile(String path) {
@ -188,32 +264,49 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override @Override
public boolean delete(String path) { public boolean delete(String path) {
s3.deleteObject(bucket, path);
String prefix = path;
if (!path.endsWith(DELIMITER)) {
prefix = path + DELIMITER;
}
try { try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) if (isDirectory(path)) {
.withPrefix(prefix).withDelimiter(DELIMITER); if (!path.endsWith(DELIMITER)) {
ObjectListing objectListing = s3.listObjects(listObjectsRequest); path += DELIMITER;
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith(DELIMITER)) {
s3.deleteObject(bucket, key);
} }
} deleteDirectory(path);
for (String pre : objectListing.getCommonPrefixes()) { } else {
delete(pre); deleteFile(path);
} }
} catch (Exception e) { } catch (Exception e) {
LogKit.error(e.getMessage(), e); LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage());
return false;
} }
s3.deleteObject(bucket, prefix);
return true; return true;
} }
private void deleteFile(String path) throws Exception {
s3.deleteObject(bucket, path);
}
private void deleteDirectory(String path) throws Exception {
List<String> files = new ArrayList<>();
for (FineFileEntry fineFileEntry : listEntry(path)) {
if (fineFileEntry.isDirectory()) {
deleteDirectory(fineFileEntry.getPath());
} else {
files.add(fineFileEntry.getPath());
}
}
deleteFiles(files);
deleteFile(path);
}
private void deleteFiles(List<String> paths) throws Exception {
//最多只能同时指定1000个key
for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) {
int toIndex = Math.min(i + PAGE_SIZE, paths.size());
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket)
.withKeys(paths.subList(i, toIndex).toArray(new String[0]));
s3.deleteObjects(deleteObjectsRequest);
}
}
@Override @Override
public boolean exist(String path) { public boolean exist(String path) {
return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path); return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path);
@ -268,51 +361,46 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER); .withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest); ObjectListing objectListing = s3.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { collectFileName(dir, result, objectListing);
while (objectListing.isTruncated()) {
objectListing = s3.listNextBatchOfObjects(objectListing);
collectFileName(dir, result, objectListing);
}
if (filter != null) {
return result.stream().filter(filter::accept).toArray(String[]::new);
}
return result.toArray(new String[0]);
}
private void collectFileName(String dir, List<String> result, ObjectListing objectListing) {
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
String key = objectSummary.getKey(); String key = objectSummary.getKey();
if (StringKit.equals(key, dir)) { if (StringKit.equals(key, dir)) {
continue; continue;
} }
String[] arr = key.split(DELIMITER); result.add(key.substring(key.lastIndexOf(DELIMITER) + 1));
String name = arr[arr.length - 1];
if (filter == null) {
result.add(name);
} else {
if (filter.accept(name)) {
result.add(name);
}
}
} }
for (String prefix : objectListing.getCommonPrefixes()) { for (String prefix : objectListing.getCommonPrefixes()) {
String[] arr = prefix.split(DELIMITER); if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) {
String name = arr[arr.length - 1] + DELIMITER; String[] arr = prefix.split(DELIMITER);
if (filter == null) { String name = arr[arr.length - 1] + DELIMITER;
result.add(name); result.add(name);
} else {
if (filter.accept(name)) {
result.add(name);
}
} }
} }
return result.toArray(new String[0]);
} }
@Override @Override
public boolean isDirectory(String path) { public boolean isDirectory(String path) {
if (path.endsWith(DELIMITER)) {
if (path.endsWith(DELIMITER) && exist(path)) { return exist(path);
return true;
}
ObjectListing listing = s3.listObjects(bucket, path);
if (listing.getObjectSummaries().isEmpty()) {
return false;
}
if (listing.getObjectSummaries().size() > 1) {
return true;
} else { } else {
S3ObjectSummary summary = listing.getObjectSummaries().get(0); ObjectListing listing = s3.listObjects(bucket, path);
return !StringKit.equals(listing.getPrefix(), summary.getKey()); List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
if (objectSummaries.isEmpty()) {
return false;
}
String dirFormat = path + DELIMITER;
return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
} }
} }
@ -322,6 +410,8 @@ public class S3ResourceRepository extends BaseResourceRepository {
S3Object s3Object = s3.getObject(bucket, path); S3Object s3Object = s3.getObject(bucket, path);
if (s3Object != null) { if (s3Object != null) {
try { try {
//s3Object要全部读完,否则会有警告
IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream());
return s3Object.getObjectMetadata().getLastModified().getTime(); return s3Object.getObjectMetadata().getLastModified().getTime();
} finally { } finally {
s3Object.close(); s3Object.close();
@ -401,27 +491,4 @@ public class S3ResourceRepository extends BaseResourceRepository {
return entry; return entry;
} }
/**
* 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt}
* 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]}
*/
private List<String> getAllNecessaryPath(String path) {
// 获取所有path路径及其所有父路径
List<String> allPath = new ArrayList<>();
allPath.add(path);
int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
// 以/结尾的先把末尾的/去掉
if (lastIdxOfDelimiter == path.length() - 1) {
path = path.substring(0, lastIdxOfDelimiter);
lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
}
while (lastIdxOfDelimiter > 0) {
allPath.add(path.substring(0, lastIdxOfDelimiter + 1));
path = path.substring(0, lastIdxOfDelimiter);
lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
}
return allPath;
}
} }

312
src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js

File diff suppressed because one or more lines are too long
Loading…
Cancel
Save