Browse Source

Pull request #463: 10.0.19.34版本插件代码合并

Merge in PG/plugin-repository-s3 from release/10.0 to persist/10.0

* commit 'aecbffaa354e8f7256bfe4ae111f06eedb756f3b':
  无jira,回退错误提升的jartime
  无jira任务 fix: 提升插件版本
  JSY-28624 fix: 优化下分片上传逻辑
  REPORT-102146 fix: isDirectory方法某些场景判断不正确
  REPORT-100817 fix: s3插件对于分片上传的支持
  REPORT-99648 fix:s3 fineui兼容问题
  无jira任务 chore: 提升版本
  无jira任务 fix: 测试连接后释放资源;list接口过滤自身目录
  REPORT-97765 feat: s3插件支持配置连接池大小
  REPORT-97765 feat: s3插件支持配置连接池大小
  REPORT-96529 fix: list接口数量限制;删除接口性能优化
  无jira任务提升jartime和版本号
  REPORT-95454 bugfix:http配置根据endpoint进行设置
persist/10.0
Icey.Zhang-张洁 1 year ago
parent
commit
3649bfc1ec
  1. 2
      .gitignore
  2. 318
      front/bundle.js
  3. 10
      front/gulpfile.js
  4. 13
      front/package.json
  5. 4
      plugin.xml
  6. 15
      src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java
  7. 18
      src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java
  8. 207
      src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
  9. 312
      src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js

2
.gitignore vendored

@ -8,3 +8,5 @@ build
local.properties
classes/
transform-classes/
node_modules
yarn.lock

318
front/bundle.js

@ -0,0 +1,318 @@
BI.config("dec.constant.intelligence.cluster.file.server", function (items) {
items.push({
value: "S3", // 地址栏显示的hash值
id: "decision-intelligence-cluster-file-s3", // id
text: "S3", // 文字
cardType: "dec.intelligence.cluster.file.s3",
workRoot: false,
});
return items;
});
!(function () {
var LABEL_WIDTH = 107, EDITOR_WIDTH = 393;
var S3 = BI.inherit(BI.Widget, {
props: {
baseCls: "dec-cluster-ftp",
value: {},
},
_store: function () {
return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", {
value: this.options.value,
});
},
watch: {
isOpen: function (v) {
this.toggle.setVisible(v);
}
},
render: function () {
var self = this, o = this.options;
return {
type: "bi.vertical",
tgap: 15,
items: [
{
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: BI.i18nText("Plugin-S3_End_Point"),
value: this.model.endPoint,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.endPointRow = _ref;
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setEndPoint(this.getValue());
},
}],
}, {
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: BI.i18nText("Plugin-S3_Region"),
value: this.model.region,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.regionRow = _ref;
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setRegion(this.getValue());
},
}],
}, {
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: BI.i18nText("Plugin-S3_Access_Key_Id"),
value: this.model.accessKeyId,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.portRow = _ref;
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setAccessKeyId(this.getValue());
},
}],
},
{
type: "dec.common.cipher.editor",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"),
text: BI.i18nText("Plugin-S3_Access_Key_Secret"),
value: this.model.password,
el: {
disabled: !o.editable,
},
ref: function (_ref) {
self.passwordRow = _ref;
},
},
{
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Bucket"),
text: BI.i18nText("Plugin-S3_Bucket"),
value: this.model.bucket,
el: {
disabled: !o.editable,
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setBucket(this.getValue());
},
}],
},
{
type: "dec.label.editor.item",
el: {
disabled: !o.editable,
},
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Dec-Please_Input"),
text: BI.i18nText("Dec-Basic_Path"),
value: this.model.workRoot,
ref: function (_ref) {
self.filePathRow = _ref;
},
}, {
type: "bi.vertical_adapt",
items: [{
type: "bi.icon_change_button",
iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font",
ref: (_ref) => {
this.OtherConfigButton = _ref;
},
handler: () => {
this.store.setIsOpen(!this.model.isOpen);
this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
}
}, {
type: "bi.text_button",
text: BI.i18nText('Plugin-S3_Other_Config'),
handler: () => {
this.store.setIsOpen(!this.model.isOpen);
this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font");
}
}]
}, {
type: 'bi.vertical',
ref: (_ref) => {
this.toggle = _ref;
},
invisible: () => !this.model.isOpen,
items: [{
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: "PathStyleAccess",
value: this.model.enablePathStyleAccess,
ref: function (_ref) {
self.enablePathStyleAccessRow = _ref;
},
el: {
disabled: !o.editable,
},
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setEnablePathStyleAccess(this.getValue());
}
}]
}, {
type: "dec.label.editor.item",
textWidth: LABEL_WIDTH,
editorWidth: EDITOR_WIDTH,
watermark: BI.i18nText("Plugin-S3_Input"),
text: "SignerOverride",
value: this.model.signerOverride,
el: {
disabled: !o.editable,
},
tgap: 15,
listeners: [{
eventName: BI.Editor.EVENT_CHANGE,
action: function () {
self.store.setSignerOverride(this.getValue());
}
}]
}],
}
]
};
},
getValue: function () {
var enablePathStyleAccess = false;
if (this.model.enablePathStyleAccess === 'true') {
enablePathStyleAccess = true;
}
return {
endPoint: this.model.endPoint,
region: this.model.region,
accessKeyId: this.model.accessKeyId,
password: this.passwordRow.getCipher(),
bucket: this.model.bucket,
workRoot: this.filePathRow.getValue(),
enablePathStyleAccess,
signerOverride: this.model.signerOverride,
};
},
validation: function () {
var valid = true;
var path = this.filePathRow.getValue();
if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) {
this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text"));
valid = false;
}
if (BI.startWith(path, "/") || !BI.endWith(path, "/")) {
this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash"));
valid = false;
}
if (!BI.isKey(path)) {
this.filePathRow.showError(BI.i18nText("Dec-Error_Null"));
valid = false;
}
if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) {
this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip"));
valid = false;
}
return valid;
},
});
BI.shortcut("dec.intelligence.cluster.file.s3", S3);
}());
!(function () {
var Model = BI.inherit(Fix.Model, {
state: function () {
var val = this.options.value;
return {
endPoint: val.endPoint,
region: val.region,
accessKeyId: val.accessKeyId,
password: val.password,
bucket: val.bucket,
workRoot: val.workRoot,
isOpen: false,
enablePathStyleAccess: 'false',
signerOverride: "",
};
},
computed: {
encodingArray: function () {
return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) {
return {
value: v,
};
});
},
},
actions: {
setEndPoint: function (v) {
this.model.endPoint = v;
},
setRegion: function (v) {
this.model.region = v;
},
setAccessKeyId: function (v) {
this.model.accessKeyId = v;
},
setBucket: function (v) {
this.model.bucket = v;
},
setEnablePathStyleAccess: function (v) {
this.model.enablePathStyleAccess = v;
},
setSignerOverride: function (v) {
this.model.signerOverride = v;
},
setIsOpen: function (v) {
this.model.isOpen = v;
},
},
});
BI.model("dec.model.intelligence.cluster.file.s3", Model);
}());

10
front/gulpfile.js

@ -0,0 +1,10 @@
const gulp = require('gulp');
const uglify = require('gulp-uglify');
function build() {
return gulp.src('./bundle.js')
.pipe(uglify())
.pipe(gulp.dest('../src/main/resources/com/fanruan/fs/s3/repository/web/js'));
}
exports.default = build;

13
front/package.json

@ -0,0 +1,13 @@
{
"name": "front",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"scripts": {
"build": "gulp"
},
"dependencies": {
"gulp": "^4.0.2",
"gulp-uglify": "^3.0.2"
}
}

4
plugin.xml

@ -5,12 +5,14 @@
<main-package>com.fanruan.fs</main-package>
<active>yes</active>
<hidden>no</hidden>
<version>1.3.7</version>
<version>1.4.0</version>
<env-version>10.0~10.0</env-version>
<jartime>2021-03-11</jartime>
<vendor>richie</vendor>
<description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
<change-notes><![CDATA[
[2023-08-08]支持分片上传,模板保存问题修复。 <br/>
[2023-06-30]修复默认配置获取错误的问题,过滤有问题的路径。 <br/>
[2023-03-28]第三方组件升级。 <br/>
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/>
[2022-09-22]第三方组件升级。 <br/>

15
src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java

@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig {
@Identifier("signerOverride")
private Conf<String> signerOverride = HolderKit.simple(StringUtils.EMPTY);
@Identifier("maxConnections")
private Conf<Integer> maxConnections = HolderKit.simple(200);
@GetConfig("endPoint")
public String getEndPoint() {
return endPoint.get();
@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig {
this.signerOverride.set(signerOverride);
}
@GetConfig("maxConnections")
public int getMaxConnections() {
return maxConnections.get();
}
@SetConfig("maxConnections")
public void setMaxConnections(int maxConnections) {
this.maxConnections.set(maxConnections);
}
@Override
public void update(String key) {
super.update(key);
@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig {
this.setBucket(newConfig.getBucket());
this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess());
this.setSignerOverride(newConfig.getSignerOverride());
this.setMaxConnections(newConfig.getMaxConnections());
}
}
@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig {
cloned.bucket = (Conf<String>) bucket.clone();
cloned.enablePathStyleAccess = (Conf<Boolean>) enablePathStyleAccess.clone();
cloned.signerOverride = (Conf<String>) signerOverride.clone();
cloned.maxConnections = (Conf<Integer>) maxConnections.clone();
return cloned;
}
}

18
src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java

@ -13,6 +13,8 @@ import com.fr.io.context.info.RepositoryProfile;
import com.fr.io.repository.ResourceRepository;
import com.fr.stable.StringUtils;
import static com.fanruan.fs.s3.repository.core.S3ResourceRepository.HTTP;
/**
* @author richie
* @version 10.0
@ -38,25 +40,33 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory<S3Config> {
@Override
public boolean verifyConfig(S3Config config) {
AmazonS3 s3 = null;
try {
BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword());
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion()))
.withCredentials(new AWSStaticCredentialsProvider(credentials));
.withCredentials(new AWSStaticCredentialsProvider(credentials)).disableChunkedEncoding();
if (config.isEnablePathStyleAccess()) {
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
}
if (StringUtils.isNotEmpty(config.getSignerOverride())) {
ClientConfiguration clientConfiguration = new ClientConfiguration();
if (StringUtils.isNotEmpty(config.getSignerOverride())) {
clientConfiguration.setSignerOverride(config.getSignerOverride());
}
LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
if (config.getEndPoint().startsWith(HTTP)) {
clientConfiguration.setProtocol(Protocol.HTTP);
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
}
AmazonS3 s3 = amazonS3ClientBuilder.build();
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
s3 = amazonS3ClientBuilder.build();
s3.listObjects(config.getBucket());
} catch (Exception e) {
LogKit.error(e.getMessage(), e);
return false;
} finally {
if (s3 != null) {
s3.shutdown();
}
}
return true;
}

207
src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

@ -7,22 +7,33 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.amazonaws.util.IOUtils;
import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit;
import com.fr.io.repository.FineFileEntry;
import com.fr.io.repository.base.BaseResourceRepository;
import com.fr.io.utils.ResourceIOUtils;
import com.fr.stable.Filter;
import com.fr.stable.StringUtils;
import com.fr.third.org.apache.commons.io.output.NullOutputStream;
import com.fr.workspace.resource.ResourceIOException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
@ -36,7 +47,14 @@ import java.util.List;
*/
public class S3ResourceRepository extends BaseResourceRepository {
private static final int PAGE_SIZE = 1000;
private static final int PART_SIZE = 5 * 1024 * 1024;
private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE;
private static final String DELIMITER = "/";
public static final String HTTP = "http:";
private final AmazonS3 s3;
private final String bucket;
@ -52,10 +70,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
}
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setMaxConnections(config.getMaxConnections());
LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections());
if (StringUtils.isNotEmpty(config.getSignerOverride())) {
clientConfiguration.setSignerOverride(config.getSignerOverride());
}
LogKit.debug("[S3] endpoint is {}", config.getEndPoint());
if (config.getEndPoint().startsWith(HTTP)) {
clientConfiguration.setProtocol(Protocol.HTTP);
}
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration);
this.s3 = amazonS3ClientBuilder.build();
this.bucket = config.getBucket();
@ -87,15 +110,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest);
collectFileEntry(result, objectListing);
collectFileEntry(dir, result, objectListing);
while (objectListing.isTruncated()) {
objectListing = s3.listNextBatchOfObjects(objectListing);
collectFileEntry(result, objectListing);
collectFileEntry(dir, result, objectListing);
}
return result.toArray(new FineFileEntry[0]);
}
private void collectFileEntry(List<FineFileEntry> result, ObjectListing objectListing) {
private void collectFileEntry(String dir, List<FineFileEntry> result, ObjectListing objectListing) {
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith(DELIMITER)) {
@ -103,11 +126,13 @@ public class S3ResourceRepository extends BaseResourceRepository {
}
}
for (String prefix : objectListing.getCommonPrefixes()) {
if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) {
FineFileEntry entry = new FineFileEntry(prefix);
entry.setDirectory(true);
result.add(entry);
}
}
}
@Override
public URL getResource(String path) {
@ -120,12 +145,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
return s3.getObject(request).getObjectContent();
} catch (Exception e) {
LogKit.error("[S3] Failed to read file {}", filePath);
LogKit.error(e.getMessage(), e);
return new ByteArrayInputStream(new byte[0]);
}
}
@Override
public void write(String path, byte[] data) {
int length = data.length;
if (length > MULTIPART_UPLOAD_LIMIT) {
multipartUpload(path, new ByteArrayInputStream(data));
} else {
ObjectMetadata metadata;
try {
metadata = s3.getObjectMetadata(bucket, path);
@ -137,11 +168,56 @@ public class S3ResourceRepository extends BaseResourceRepository {
}
}
if (metadata != null) {
metadata.setContentLength(data.length);
metadata.setContentLength(length);
}
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata);
}
}
@Override
public void write(String path, InputStream inputStream) throws ResourceIOException {
multipartUpload(path, inputStream);
}
private void multipartUpload(String path, InputStream inputStream) {
try {
// Step 1: 初始化分片上传
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path);
InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest);
String uploadId = initResponse.getUploadId();
// Step 2: 分片上传文件
List<PartETag> partETags = new ArrayList<>();
byte[] buffer = new byte[PART_SIZE];
int bytesRead;
int partNumber = 1;
while ((bytesRead = inputStream.read(buffer)) > 0) {
// 创建上传请求
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucket)
.withKey(path)
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead))
.withPartSize(bytesRead);
// 上传分片
UploadPartResult uploadResult = s3.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
partNumber++;
}
// Step 3: 完成分片上传
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags);
s3.completeMultipartUpload(compRequest);
} catch (IOException e) {
throw new ResourceIOException(e);
} finally {
ResourceIOUtils.close(inputStream);
}
}
@Override
public boolean createFile(String path) {
@ -188,30 +264,47 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean delete(String path) {
try {
if (isDirectory(path)) {
if (!path.endsWith(DELIMITER)) {
path += DELIMITER;
}
deleteDirectory(path);
} else {
deleteFile(path);
}
} catch (Exception e) {
LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage());
return false;
}
return true;
}
private void deleteFile(String path) throws Exception {
s3.deleteObject(bucket, path);
String prefix = path;
if (!path.endsWith(DELIMITER)) {
prefix = path + DELIMITER;
}
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(prefix).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest);
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith(DELIMITER)) {
s3.deleteObject(bucket, key);
private void deleteDirectory(String path) throws Exception {
List<String> files = new ArrayList<>();
for (FineFileEntry fineFileEntry : listEntry(path)) {
if (fineFileEntry.isDirectory()) {
deleteDirectory(fineFileEntry.getPath());
} else {
files.add(fineFileEntry.getPath());
}
}
for (String pre : objectListing.getCommonPrefixes()) {
delete(pre);
deleteFiles(files);
deleteFile(path);
}
} catch (Exception e) {
LogKit.error(e.getMessage(), e);
private void deleteFiles(List<String> paths) throws Exception {
//最多只能同时指定1000个key
for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) {
int toIndex = Math.min(i + PAGE_SIZE, paths.size());
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket)
.withKeys(paths.subList(i, toIndex).toArray(new String[0]));
s3.deleteObjects(deleteObjectsRequest);
}
s3.deleteObject(bucket, prefix);
return true;
}
@Override
@ -268,51 +361,46 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
collectFileName(dir, result, objectListing);
while (objectListing.isTruncated()) {
objectListing = s3.listNextBatchOfObjects(objectListing);
collectFileName(dir, result, objectListing);
}
if (filter != null) {
return result.stream().filter(filter::accept).toArray(String[]::new);
}
return result.toArray(new String[0]);
}
private void collectFileName(String dir, List<String> result, ObjectListing objectListing) {
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
String key = objectSummary.getKey();
if (StringKit.equals(key, dir)) {
continue;
}
String[] arr = key.split(DELIMITER);
String name = arr[arr.length - 1];
if (filter == null) {
result.add(name);
} else {
if (filter.accept(name)) {
result.add(name);
}
}
result.add(key.substring(key.lastIndexOf(DELIMITER) + 1));
}
for (String prefix : objectListing.getCommonPrefixes()) {
if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) {
String[] arr = prefix.split(DELIMITER);
String name = arr[arr.length - 1] + DELIMITER;
if (filter == null) {
result.add(name);
} else {
if (filter.accept(name)) {
result.add(name);
}
}
}
return result.toArray(new String[0]);
}
@Override
public boolean isDirectory(String path) {
if (path.endsWith(DELIMITER) && exist(path)) {
return true;
}
if (path.endsWith(DELIMITER)) {
return exist(path);
} else {
ObjectListing listing = s3.listObjects(bucket, path);
if (listing.getObjectSummaries().isEmpty()) {
List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
if (objectSummaries.isEmpty()) {
return false;
}
if (listing.getObjectSummaries().size() > 1) {
return true;
} else {
S3ObjectSummary summary = listing.getObjectSummaries().get(0);
return !StringKit.equals(listing.getPrefix(), summary.getKey());
String dirFormat = path + DELIMITER;
return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat));
}
}
@ -322,6 +410,8 @@ public class S3ResourceRepository extends BaseResourceRepository {
S3Object s3Object = s3.getObject(bucket, path);
if (s3Object != null) {
try {
//s3Object要全部读完,否则会有警告
IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream());
return s3Object.getObjectMetadata().getLastModified().getTime();
} finally {
s3Object.close();
@ -401,27 +491,4 @@ public class S3ResourceRepository extends BaseResourceRepository {
return entry;
}
/**
* 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt}
* 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]}
*/
private List<String> getAllNecessaryPath(String path) {
// 获取所有path路径及其所有父路径
List<String> allPath = new ArrayList<>();
allPath.add(path);
int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
// 以/结尾的先把末尾的/去掉
if (lastIdxOfDelimiter == path.length() - 1) {
path = path.substring(0, lastIdxOfDelimiter);
lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
}
while (lastIdxOfDelimiter > 0) {
allPath.add(path.substring(0, lastIdxOfDelimiter + 1));
path = path.substring(0, lastIdxOfDelimiter);
lastIdxOfDelimiter = path.lastIndexOf(DELIMITER);
}
return allPath;
}
}

312
src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js

File diff suppressed because one or more lines are too long
Loading…
Cancel
Save