diff --git a/.gitignore b/.gitignore
index 29c9cac..256375a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,4 +7,6 @@ target/
build
local.properties
classes/
-transform-classes/
\ No newline at end of file
+transform-classes/
+node_modules
+yarn.lock
\ No newline at end of file
diff --git a/front/bundle.js b/front/bundle.js
new file mode 100644
index 0000000..bdfc9ed
--- /dev/null
+++ b/front/bundle.js
@@ -0,0 +1,318 @@
+BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
+ items.push({
+ value: "S3", // 地址栏显示的hash值
+ id: "decision-intelligence-cluster-file-s3", // id
+ text: "S3", // 文字
+ cardType: "dec.intelligence.cluster.file.s3",
+ workRoot: false,
+ });
+
+ return items;
+});
+
+
+!(function () {
+ var LABEL_WIDTH = 107, EDITOR_WIDTH = 393;
+ var S3 = BI.inherit(BI.Widget, {
+
+ props: {
+ baseCls: "dec-cluster-ftp",
+ value: {},
+ },
+
+ _store: function () {
+ return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", {
+ value: this.options.value,
+ });
+ },
+
+ watch: {
+ isOpen: function (v) {
+ this.toggle.setVisible(v);
+ }
+ },
+
+ render: function () {
+ var self = this, o = this.options;
+
+ return {
+ type: "bi.vertical",
+ tgap: 15,
+ items: [
+ {
+ type: "dec.label.editor.item",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Input"),
+ text: BI.i18nText("Plugin-S3_End_Point"),
+ value: this.model.endPoint,
+ el: {
+ disabled: !o.editable,
+ },
+ ref: function (_ref) {
+ self.endPointRow = _ref;
+ },
+ listeners: [{
+ eventName: BI.Editor.EVENT_CHANGE,
+ action: function () {
+ self.store.setEndPoint(this.getValue());
+ },
+ }],
+ }, {
+ type: "dec.label.editor.item",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Input"),
+ text: BI.i18nText("Plugin-S3_Region"),
+ value: this.model.region,
+ el: {
+ disabled: !o.editable,
+ },
+ ref: function (_ref) {
+ self.regionRow = _ref;
+ },
+ listeners: [{
+ eventName: BI.Editor.EVENT_CHANGE,
+ action: function () {
+ self.store.setRegion(this.getValue());
+ },
+ }],
+ }, {
+ type: "dec.label.editor.item",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Input"),
+ text: BI.i18nText("Plugin-S3_Access_Key_Id"),
+ value: this.model.accessKeyId,
+ el: {
+ disabled: !o.editable,
+ },
+ ref: function (_ref) {
+ self.portRow = _ref;
+ },
+ listeners: [{
+ eventName: BI.Editor.EVENT_CHANGE,
+ action: function () {
+ self.store.setAccessKeyId(this.getValue());
+ },
+ }],
+ },
+ {
+ type: "dec.common.cipher.editor",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"),
+ text: BI.i18nText("Plugin-S3_Access_Key_Secret"),
+ value: this.model.password,
+ el: {
+ disabled: !o.editable,
+ },
+ ref: function (_ref) {
+ self.passwordRow = _ref;
+ },
+ },
+ {
+ type: "dec.label.editor.item",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Bucket"),
+ text: BI.i18nText("Plugin-S3_Bucket"),
+ value: this.model.bucket,
+ el: {
+ disabled: !o.editable,
+ },
+ listeners: [{
+ eventName: BI.Editor.EVENT_CHANGE,
+ action: function () {
+ self.store.setBucket(this.getValue());
+ },
+ }],
+ },
+ {
+ type: "dec.label.editor.item",
+ el: {
+ disabled: !o.editable,
+ },
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Dec-Please_Input"),
+ text: BI.i18nText("Dec-Basic_Path"),
+ value: this.model.workRoot,
+ ref: function (_ref) {
+ self.filePathRow = _ref;
+ },
+ }, {
+ type: "bi.vertical_adapt",
+ items: [{
+ type: "bi.icon_change_button",
+ iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font",
+ ref: (_ref) => {
+ this.OtherConfigButton = _ref;
+ },
+ handler: () => {
+ this.store.setIsOpen(!this.model.isOpen);
+ this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
+ }
+ }, {
+ type: "bi.text_button",
+ text: BI.i18nText('Plugin-S3_Other_Config'),
+ handler: () => {
+ this.store.setIsOpen(!this.model.isOpen);
+ this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
+ }
+ }]
+ }, {
+ type: 'bi.vertical',
+ ref: (_ref) => {
+ this.toggle = _ref;
+ },
+ invisible: () => !this.model.isOpen,
+ items: [{
+ type: "dec.label.editor.item",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Input"),
+ text: "PathStyleAccess",
+ value: this.model.enablePathStyleAccess,
+ ref: function (_ref) {
+ self.enablePathStyleAccessRow = _ref;
+ },
+ el: {
+ disabled: !o.editable,
+ },
+ listeners: [{
+ eventName: BI.Editor.EVENT_CHANGE,
+ action: function () {
+ self.store.setEnablePathStyleAccess(this.getValue());
+ }
+ }]
+ }, {
+ type: "dec.label.editor.item",
+ textWidth: LABEL_WIDTH,
+ editorWidth: EDITOR_WIDTH,
+ watermark: BI.i18nText("Plugin-S3_Input"),
+ text: "SignerOverride",
+ value: this.model.signerOverride,
+ el: {
+ disabled: !o.editable,
+ },
+ tgap: 15,
+ listeners: [{
+ eventName: BI.Editor.EVENT_CHANGE,
+ action: function () {
+ self.store.setSignerOverride(this.getValue());
+ }
+ }]
+ }],
+ }
+ ]
+ };
+ },
+
+ getValue: function () {
+ var enablePathStyleAccess = false;
+ if (this.model.enablePathStyleAccess === 'true') {
+ enablePathStyleAccess = true;
+ }
+
+ return {
+ endPoint: this.model.endPoint,
+ region: this.model.region,
+ accessKeyId: this.model.accessKeyId,
+ password: this.passwordRow.getCipher(),
+ bucket: this.model.bucket,
+ workRoot: this.filePathRow.getValue(),
+ enablePathStyleAccess,
+ signerOverride: this.model.signerOverride,
+ };
+ },
+
+ validation: function () {
+ var valid = true;
+ var path = this.filePathRow.getValue();
+ if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) {
+ this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text"));
+ valid = false;
+ }
+ if (BI.startWith(path, "/") || !BI.endWith(path, "/")) {
+ this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash"));
+ valid = false;
+ }
+ if (!BI.isKey(path)) {
+ this.filePathRow.showError(BI.i18nText("Dec-Error_Null"));
+ valid = false;
+ }
+ if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) {
+ this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip"));
+ valid = false;
+ }
+
+ return valid;
+ },
+
+ });
+ BI.shortcut("dec.intelligence.cluster.file.s3", S3);
+}());
+
+
+!(function () {
+ var Model = BI.inherit(Fix.Model, {
+
+ state: function () {
+ var val = this.options.value;
+
+ return {
+ endPoint: val.endPoint,
+ region: val.region,
+ accessKeyId: val.accessKeyId,
+ password: val.password,
+ bucket: val.bucket,
+ workRoot: val.workRoot,
+ isOpen: false,
+ enablePathStyleAccess: 'false',
+ signerOverride: "",
+ };
+ },
+
+ computed: {
+ encodingArray: function () {
+ return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) {
+ return {
+ value: v,
+ };
+ });
+ },
+ },
+
+ actions: {
+ setEndPoint: function (v) {
+ this.model.endPoint = v;
+ },
+
+ setRegion: function (v) {
+ this.model.region = v;
+ },
+
+ setAccessKeyId: function (v) {
+ this.model.accessKeyId = v;
+ },
+
+ setBucket: function (v) {
+ this.model.bucket = v;
+ },
+
+ setEnablePathStyleAccess: function (v) {
+ this.model.enablePathStyleAccess = v;
+ },
+
+ setSignerOverride: function (v) {
+ this.model.signerOverride = v;
+ },
+
+ setIsOpen: function (v) {
+ this.model.isOpen = v;
+ },
+ },
+ });
+ BI.model("dec.model.intelligence.cluster.file.s3", Model);
+}());
diff --git a/front/gulpfile.js b/front/gulpfile.js
new file mode 100644
index 0000000..bc6eefa
--- /dev/null
+++ b/front/gulpfile.js
@@ -0,0 +1,10 @@
+const gulp = require('gulp');
+const uglify = require('gulp-uglify');
+
+function build() {
+ return gulp.src('./bundle.js')
+ .pipe(uglify())
+ .pipe(gulp.dest('../src/main/resources/com/fanruan/fs/s3/repository/web/js'));
+}
+
+exports.default = build;
\ No newline at end of file
diff --git a/front/package.json b/front/package.json
new file mode 100644
index 0000000..10e2932
--- /dev/null
+++ b/front/package.json
@@ -0,0 +1,13 @@
+{
+ "name": "front",
+ "version": "1.0.0",
+ "main": "index.js",
+ "license": "MIT",
+ "scripts": {
+ "build": "gulp"
+ },
+ "dependencies": {
+ "gulp": "^4.0.2",
+ "gulp-uglify": "^3.0.2"
+ }
+}
diff --git a/plugin.xml b/plugin.xml
index e664cbd..030424b 100644
--- a/plugin.xml
+++ b/plugin.xml
@@ -5,12 +5,14 @@
com.fanruan.fs
yes
no
- 1.3.7
+ 1.4.0
10.0~10.0
2021-03-11
richie
+ [2023-06-30]修复默认配置获取错误的问题，过滤有问题的路径。
[2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。
[2022-09-22]第三方组件升级。
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java
index 5821864..976a9b8 100644
--- a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java
+++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java
@@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig {
@Identifier("signerOverride")
private Conf signerOverride = HolderKit.simple(StringUtils.EMPTY);
+ @Identifier("maxConnections")
+ private Conf maxConnections = HolderKit.simple(200);
+
@GetConfig("endPoint")
public String getEndPoint() {
return endPoint.get();
@@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig {
this.signerOverride.set(signerOverride);
}
+ @GetConfig("maxConnections")
+ public int getMaxConnections() {
+ return maxConnections.get();
+ }
+
+ @SetConfig("maxConnections")
+ public void setMaxConnections(int maxConnections) {
+ this.maxConnections.set(maxConnections);
+ }
+
@Override
public void update(String key) {
super.update(key);
@@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig {
this.setBucket(newConfig.getBucket());
this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess());
this.setSignerOverride(newConfig.getSignerOverride());
+ this.setMaxConnections(newConfig.getMaxConnections());
}
}
@@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig {
cloned.bucket = (Conf) bucket.clone();
cloned.enablePathStyleAccess = (Conf) enablePathStyleAccess.clone();
cloned.signerOverride = (Conf) signerOverride.clone();
+ cloned.maxConnections = (Conf) maxConnections.clone();
return cloned;
}
}
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java
index 560d81e..5de506c 100644
--- a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java
+++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java
@@ -13,6 +13,8 @@ import com.fr.io.context.info.RepositoryProfile;
import com.fr.io.repository.ResourceRepository;
import com.fr.stable.StringUtils;
+import static com.fanruan.fs.s3.repository.core.S3ResourceRepository.HTTP;
+
/**
* @author richie
* @version 10.0
@@ -38,25 +40,33 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory {
@Override
public boolean verifyConfig(S3Config config) {
+ AmazonS3 s3 = null;
try {
BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword());
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion()))
- .withCredentials(new AWSStaticCredentialsProvider(credentials));
+ .withCredentials(new AWSStaticCredentialsProvider(credentials)).disableChunkedEncoding();
if (config.isEnablePathStyleAccess()) {
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
}
+ ClientConfiguration clientConfiguration = new ClientConfiguration();
if (StringUtils.isNotEmpty(config.getSignerOverride())) {
- ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setSignerOverride(config.getSignerOverride());
+ }
+ LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
+ if (config.getEndPoint().startsWith(HTTP)) {
clientConfiguration.setProtocol(Protocol.HTTP);
- amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
}
- AmazonS3 s3 = amazonS3ClientBuilder.build();
+ amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
+ s3 = amazonS3ClientBuilder.build();
s3.listObjects(config.getBucket());
} catch (Exception e) {
LogKit.error(e.getMessage(), e);
return false;
+ } finally {
+ if (s3 != null) {
+ s3.shutdown();
+ }
}
return true;
}
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
index 66f907e..f82d7ad 100644
--- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
+++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
@@ -7,22 +7,33 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
+import com.amazonaws.util.IOUtils;
import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
+import com.fr.io.utils.ResourceIOUtils;
import com.fr.stable.Filter;
import com.fr.stable.StringUtils;
+import com.fr.third.org.apache.commons.io.output.NullOutputStream;
import com.fr.workspace.resource.ResourceIOException;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
@@ -36,7 +47,14 @@ import java.util.List;
*/
public class S3ResourceRepository extends BaseResourceRepository {
+ private static final int PAGE_SIZE = 1000;
+
+ private static final int PART_SIZE = 5 * 1024 * 1024;
+
+ private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
+
private static final String DELIMITER = "/";
+ public static final String HTTP = "http:";
private final AmazonS3 s3;
private final String bucket;
@@ -52,10 +70,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
}
ClientConfiguration clientConfiguration = new ClientConfiguration();
+ clientConfiguration.setMaxConnections(config.getMaxConnections());
+ LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections());
if (StringUtils.isNotEmpty(config.getSignerOverride())) {
clientConfiguration.setSignerOverride(config.getSignerOverride());
}
- clientConfiguration.setProtocol(Protocol.HTTP);
+ LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
+ if (config.getEndPoint().startsWith(HTTP)) {
+ clientConfiguration.setProtocol(Protocol.HTTP);
+ }
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
this.s3 = amazonS3ClientBuilder.build();
this.bucket = config.getBucket();
@@ -87,15 +110,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest);
- collectFileEntry(result, objectListing);
+ collectFileEntry(dir, result, objectListing);
while (objectListing.isTruncated()) {
objectListing = s3.listNextBatchOfObjects(objectListing);
- collectFileEntry(result, objectListing);
+ collectFileEntry(dir, result, objectListing);
}
return result.toArray(new FineFileEntry[0]);
}
- private void collectFileEntry(List result, ObjectListing objectListing) {
+ private void collectFileEntry(String dir, List result, ObjectListing objectListing) {
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith(DELIMITER)) {
@@ -103,9 +126,11 @@ public class S3ResourceRepository extends BaseResourceRepository {
}
}
for (String prefix : objectListing.getCommonPrefixes()) {
- FineFileEntry entry = new FineFileEntry(prefix);
- entry.setDirectory(true);
- result.add(entry);
+ if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) {
+ FineFileEntry entry = new FineFileEntry(prefix);
+ entry.setDirectory(true);
+ result.add(entry);
+ }
}
}
@@ -120,28 +145,79 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
return s3.getObject(request).getObjectContent();
} catch (Exception e) {
+ LogKit.error("[S3] Failed to read file {}", filePath);
+ LogKit.error(e.getMessage(), e);
return new ByteArrayInputStream(new byte[0]);
}
}
@Override
public void write(String path, byte[] data) {
- ObjectMetadata metadata;
- try {
- metadata = s3.getObjectMetadata(bucket, path);
- } catch (Exception e) {
- metadata = new ObjectMetadata();
- String mimeType = URLConnection.guessContentTypeFromName(path);
- if (mimeType != null) {
- metadata.setContentType(mimeType);
+ int length = data.length;
+ if (length > MULTIPART_UPLOAD_LIMIT) {
+ multipartUpload(path, new ByteArrayInputStream(data));
+ } else {
+ ObjectMetadata metadata;
+ try {
+ metadata = s3.getObjectMetadata(bucket, path);
+ } catch (Exception e) {
+ metadata = new ObjectMetadata();
+ String mimeType = URLConnection.guessContentTypeFromName(path);
+ if (mimeType != null) {
+ metadata.setContentType(mimeType);
+ }
}
+ if (metadata != null) {
+ metadata.setContentLength(length);
+ }
+ s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
- if (metadata != null) {
- metadata.setContentLength(data.length);
- }
- s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
+ @Override
+ public void write(String path, InputStream inputStream) throws ResourceIOException {
+ multipartUpload(path, inputStream);
+ }
+
+ private void multipartUpload(String path, InputStream inputStream) {
+ try {
+ // Step 1: 初始化分片上传
+ InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
+ InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
+ String uploadId = initResponse.getUploadId();
+
+ // Step 2: 分片上传文件
+ List partETags = new ArrayList<>();
+ byte[] buffer = new byte[PART_SIZE];
+ int bytesRead;
+ int partNumber = 1;
+
+ while ((bytesRead = inputStream.read(buffer)) > 0) {
+ // 创建上传请求
+ UploadPartRequest uploadRequest = new UploadPartRequest()
+ .withBucketName(bucket)
+ .withKey(path)
+ .withUploadId(uploadId)
+ .withPartNumber(partNumber)
+ .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
+ .withPartSize(bytesRead);
+
+ // 上传分片
+ UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
+ partETags.add(uploadResult.getPartETag());
+
+ partNumber++;
+ }
+
+ // Step 3: 完成分片上传
+ CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
+ s3.completeMultipartUpload(compRequest);
+ } catch (IOException e) {
+ throw new ResourceIOException(e);
+ } finally {
+ ResourceIOUtils.close(inputStream);
+ }
+ }
@Override
public boolean createFile(String path) {
@@ -188,32 +264,49 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean delete(String path) {
-
- s3.deleteObject(bucket, path);
- String prefix = path;
- if (!path.endsWith(DELIMITER)) {
- prefix = path + DELIMITER;
- }
try {
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
- .withPrefix(prefix).withDelimiter(DELIMITER);
- ObjectListing objectListing = s3.listObjects(listObjectsRequest);
- for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
- String key = summary.getKey();
- if (!key.endsWith(DELIMITER)) {
- s3.deleteObject(bucket, key);
+ if (isDirectory(path)) {
+ if (!path.endsWith(DELIMITER)) {
+ path += DELIMITER;
}
- }
- for (String pre : objectListing.getCommonPrefixes()) {
- delete(pre);
+ deleteDirectory(path);
+ } else {
+ deleteFile(path);
}
} catch (Exception e) {
- LogKit.error(e.getMessage(), e);
+ LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage());
+ return false;
}
- s3.deleteObject(bucket, prefix);
return true;
}
+ private void deleteFile(String path) throws Exception {
+ s3.deleteObject(bucket, path);
+ }
+
+ private void deleteDirectory(String path) throws Exception {
+ List files = new ArrayList<>();
+ for (FineFileEntry fineFileEntry : listEntry(path)) {
+ if (fineFileEntry.isDirectory()) {
+ deleteDirectory(fineFileEntry.getPath());
+ } else {
+ files.add(fineFileEntry.getPath());
+ }
+ }
+ deleteFiles(files);
+ deleteFile(path);
+ }
+
+ private void deleteFiles(List paths) throws Exception {
+ //最多只能同时指定1000个key
+ for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) {
+ int toIndex = Math.min(i + PAGE_SIZE, paths.size());
+ DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket)
+ .withKeys(paths.subList(i, toIndex).toArray(new String[0]));
+ s3.deleteObjects(deleteObjectsRequest);
+ }
+ }
+
@Override
public boolean exist(String path) {
return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path);
@@ -268,51 +361,46 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest);
- for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
+ collectFileName(dir, result, objectListing);
+ while (objectListing.isTruncated()) {
+ objectListing = s3.listNextBatchOfObjects(objectListing);
+ collectFileName(dir, result, objectListing);
+ }
+ if (filter != null) {
+ return result.stream().filter(filter::accept).toArray(String[]::new);
+ }
+ return result.toArray(new String[0]);
+ }
+ private void collectFileName(String dir, List result, ObjectListing objectListing) {
+ for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
String key = objectSummary.getKey();
if (StringKit.equals(key, dir)) {
continue;
}
- String[] arr = key.split(DELIMITER);
- String name = arr[arr.length - 1];
- if (filter == null) {
- result.add(name);
- } else {
- if (filter.accept(name)) {
- result.add(name);
- }
- }
+ result.add(key.substring(key.lastIndexOf(DELIMITER) + 1));
}
for (String prefix : objectListing.getCommonPrefixes()) {
- String[] arr = prefix.split(DELIMITER);
- String name = arr[arr.length - 1] + DELIMITER;
- if (filter == null) {
+ if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) {
+ String[] arr = prefix.split(DELIMITER);
+ String name = arr[arr.length - 1] + DELIMITER;
result.add(name);
- } else {
- if (filter.accept(name)) {
- result.add(name);
- }
}
}
- return result.toArray(new String[0]);
}
@Override
public boolean isDirectory(String path) {
-
- if (path.endsWith(DELIMITER) && exist(path)) {
- return true;
- }
- ObjectListing listing = s3.listObjects(bucket, path);
- if (listing.getObjectSummaries().isEmpty()) {
- return false;
- }
- if (listing.getObjectSummaries().size() > 1) {
- return true;
+ if (path.endsWith(DELIMITER)) {
+ return exist(path);
} else {
- S3ObjectSummary summary = listing.getObjectSummaries().get(0);
- return !StringKit.equals(listing.getPrefix(), summary.getKey());
+ ObjectListing listing = s3.listObjects(bucket, path);
+ List objectSummaries = listing.getObjectSummaries();
+ if (objectSummaries.isEmpty()) {
+ return false;
+ }
+ String dirFormat = path + DELIMITER;
+ return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
}
}
@@ -322,6 +410,8 @@ public class S3ResourceRepository extends BaseResourceRepository {
S3Object s3Object = s3.getObject(bucket, path);
if (s3Object != null) {
try {
+ //s3Object要全部读完，否则会有警告
+ IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream());
return s3Object.getObjectMetadata().getLastModified().getTime();
} finally {
s3Object.close();
@@ -401,27 +491,4 @@ public class S3ResourceRepository extends BaseResourceRepository {
return entry;
}
- /**
- * 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt}
- * 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]}
- */
- private List getAllNecessaryPath(String path) {
-
- // 获取所有path路径及其所有父路径
- List allPath = new ArrayList<>();
- allPath.add(path);
- int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
- // 以/结尾的先把末尾的/去掉
- if (lastIdxOfDelimiter == path.length() - 1) {
- path = path.substring(0, lastIdxOfDelimiter);
- lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
- }
- while (lastIdxOfDelimiter > 0) {
- allPath.add(path.substring(0, lastIdxOfDelimiter + 1));
- path = path.substring(0, lastIdxOfDelimiter);
- lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
- }
- return allPath;
- }
-
}
diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
index af8b1bd..b86e382 100644
--- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
+++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js
@@ -1,311 +1 @@
-BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
- items.push({
- value: "S3", // 地址栏显示的hash值
- id: "decision-intelligence-cluster-file-s3", // id
- text: "S3", // 文字
- cardType: "dec.intelligence.cluster.file.s3",
- workRoot: false,
- });
-
- return items;
-});
-
-
-!(function () {
- var LABEL_WIDTH = 107, EDITOR_WIDTH = 393;
- var S3 = BI.inherit(BI.Widget, {
-
- props: {
- baseCls: "dec-cluster-ftp",
- value: {},
- },
-
- _store: function () {
- return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", {
- value: this.options.value,
- });
- },
-
- watch: {},
-
- render: function () {
- var self = this, o = this.options;
-
- return {
- type: "bi.vertical",
- tgap: 15,
- items: [
- {
- type: "dec.label.editor.item",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Input"),
- text: BI.i18nText("Plugin-S3_End_Point"),
- value: this.model.endPoint,
- el: {
- disabled: !o.editable,
- },
- ref: function (_ref) {
- self.endPointRow = _ref;
- },
- listeners: [{
- eventName: BI.Editor.EVENT_CHANGE,
- action: function () {
- self.store.setEndPoint(this.getValue());
- },
- }],
- }, {
- type: "dec.label.editor.item",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Input"),
- text: BI.i18nText("Plugin-S3_Region"),
- value: this.model.region,
- el: {
- disabled: !o.editable,
- },
- ref: function (_ref) {
- self.regionRow = _ref;
- },
- listeners: [{
- eventName: BI.Editor.EVENT_CHANGE,
- action: function () {
- self.store.setRegion(this.getValue());
- },
- }],
- }, {
- type: "dec.label.editor.item",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Input"),
- text: BI.i18nText("Plugin-S3_Access_Key_Id"),
- value: this.model.accessKeyId,
- el: {
- disabled: !o.editable,
- },
- ref: function (_ref) {
- self.portRow = _ref;
- },
- listeners: [{
- eventName: BI.Editor.EVENT_CHANGE,
- action: function () {
- self.store.setAccessKeyId(this.getValue());
- },
- }],
- },
- {
- type: "dec.common.cipher.editor",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"),
- text: BI.i18nText("Plugin-S3_Access_Key_Secret"),
- value: this.model.password,
- el: {
- disabled: !o.editable,
- },
- ref: function (_ref) {
- self.passwordRow = _ref;
- },
- },
- {
- type: "dec.label.editor.item",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Bucket"),
- text: BI.i18nText("Plugin-S3_Bucket"),
- value: this.model.bucket,
- el: {
- disabled: !o.editable,
- },
- listeners: [{
- eventName: BI.Editor.EVENT_CHANGE,
- action: function () {
- self.store.setBucket(this.getValue());
- },
- }],
- },
- {
- type: "dec.label.editor.item",
- el: {
- disabled: !o.editable,
- },
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Dec-Please_Input"),
- text: BI.i18nText("Dec-Basic_Path"),
- value: this.model.workRoot,
- ref: function (_ref) {
- self.filePathRow = _ref;
- },
- }, {
- type: "bi.vertical_adapt",
- items: [{
- type: "bi.icon_change_button",
- iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font",
- ref: (_ref) => {
- this.OtherConfigButton = _ref;
- },
- handler: () => {
- this.store.setIsOpen(!this.model.isOpen);
- this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
- }
- }, {
- type: "bi.text_button",
- text: BI.i18nText('Plugin-S3_Other_Config'),
- handler: () => {
- this.store.setIsOpen(!this.model.isOpen);
- this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
- }
- }]
- }, {
- type: 'bi.vertical',
- invisible: () => !this.model.isOpen,
- items: [{
- type: "dec.label.editor.item",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Input"),
- text: "PathStyleAccess",
- value: this.model.enablePathStyleAccess,
- ref: function (_ref) {
- self.enablePathStyleAccessRow = _ref;
- },
- el: {
- disabled: !o.editable,
- },
- listeners: [{
- eventName: BI.Editor.EVENT_CHANGE,
- action: function () {
- self.store.setEnablePathStyleAccess(this.getValue());
- }
- }]
- }, {
- type: "dec.label.editor.item",
- textWidth: LABEL_WIDTH,
- editorWidth: EDITOR_WIDTH,
- watermark: BI.i18nText("Plugin-S3_Input"),
- text: "SignerOverride",
- value: this.model.signerOverride,
- el: {
- disabled: !o.editable,
- },
- tgap: 15,
- listeners: [{
- eventName: BI.Editor.EVENT_CHANGE,
- action: function () {
- self.store.setSignerOverride(this.getValue());
- }
- }]
- }],
- }
- ]
- };
- },
-
- getValue: function () {
- var enablePathStyleAccess = false;
- if (this.model.enablePathStyleAccess === 'true') {
- enablePathStyleAccess = true;
- }
-
- return {
- endPoint: this.model.endPoint,
- region: this.model.region,
- accessKeyId: this.model.accessKeyId,
- password: this.passwordRow.getCipher(),
- bucket: this.model.bucket,
- workRoot: this.filePathRow.getValue(),
- enablePathStyleAccess,
- signerOverride: this.model.signerOverride,
- };
- },
-
- validation: function () {
- var valid = true;
- var path = this.filePathRow.getValue();
- if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) {
- this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text"));
- valid = false;
- }
- if (BI.startWith(path, "/") || !BI.endWith(path, "/")) {
- this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash"));
- valid = false;
- }
- if (!BI.isKey(path)) {
- this.filePathRow.showError(BI.i18nText("Dec-Error_Null"));
- valid = false;
- }
- if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) {
- this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip"));
- valid = false;
- }
-
- return valid;
- },
-
- });
- BI.shortcut("dec.intelligence.cluster.file.s3", S3);
-}());
-
-
-!(function () {
- var Model = BI.inherit(Fix.Model, {
-
- state: function () {
- var val = this.options.value;
-
- return {
- endPoint: val.endPoint,
- region: val.region,
- accessKeyId: val.accessKeyId,
- password: val.password,
- bucket: val.bucket,
- workRoot: val.workRoot,
- isOpen: false,
- enablePathStyleAccess: 'false',
- signerOverride: "",
- };
- },
-
- computed: {
- encodingArray: function () {
- return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) {
- return {
- value: v,
- };
- });
- },
- },
-
- actions: {
- setEndPoint: function (v) {
- this.model.endPoint = v;
- },
-
- setRegion: function (v) {
- this.model.region = v;
- },
-
- setAccessKeyId: function (v) {
- this.model.accessKeyId = v;
- },
-
- setBucket: function (v) {
- this.model.bucket = v;
- },
-
- setEnablePathStyleAccess: function (v) {
- this.model.enablePathStyleAccess = v;
- },
-
- setSignerOverride: function (v) {
- this.model.signerOverride = v;
- },
-
- setIsOpen: function (v) {
- this.model.isOpen = v;
- },
- },
- });
- BI.model("dec.model.intelligence.cluster.file.s3", Model);
-}());
+BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},watch:{isOpen:function(e){this.toggle.setVisible(e)}},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",ref:e=>{this.toggle=e},invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:this.model.enablePathStyleAccess,ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:"false",signerOverride:""}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}();
\ No newline at end of file