fanruan
/
plugin-repository-s3
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Pull request #87: REPORT-88094 fix: 文件数量较多时，listEntry结果不全；优化写文件性能 

Merge in PG/plugin-repository-s3 from ~AFLY/plugin-repository-s3:release/10.0 to release/10.0

* commit '8f1227e659711c2c53df8afd4ab5f7e1ff13f404':
  REPORT-88094 fix: change-notes
  REPORT-88094 fix: 文件数量较多时，listEntry结果不全
  无jira任务 perf: 优化写文件性能
release/10.0
Afly-储泓飞 2 years ago
parent
e1b8a79786 8f1227e659
commit
8533a78ee9
2 changed files with 72 additions and 33 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      plugin.xml
  2. 90
      src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java

3
plugin.xml
Unescape View File

@ -5,12 +5,13 @@
<main-package>com.fanruan.fs</main-package>
<active>yes</active>
<hidden>no</hidden>
<version>1.3.5</version>
<version>1.3.6</version>
<env-version>10.0~10.0</env-version>
<jartime>2021-03-11</jartime>
<vendor>richie</vendor>
<description><![CDATA[使用支持S3协议的云存储文件系统作为文件服务器。]]></description>
<change-notes><![CDATA[
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。<br/>
[2022-09-22]第三方组件升级。 <br/>
[2022-07-25]修复http无法连接，而https和缺省的时候正常<br/>
[2022-06-30]新增enablePathStyleAccess、signerOverride后台配置<br/>

90
src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java
Unescape View File

@ -87,6 +87,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket)
.withPrefix(dir).withDelimiter(DELIMITER);
ObjectListing objectListing = s3.listObjects(listObjectsRequest);
collectFileEntry(result, objectListing);
while (objectListing.isTruncated()) {
objectListing = s3.listNextBatchOfObjects(objectListing);
collectFileEntry(result, objectListing);
}
return result.toArray(new FineFileEntry[0]);
}
private void collectFileEntry(List<FineFileEntry> result, ObjectListing objectListing) {
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith(DELIMITER)) {
@ -98,8 +107,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
entry.setDirectory(true);
result.add(entry);
}
return result.toArray(new FineFileEntry[0]);
}
@Override
@ -119,17 +126,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public void write(String path, byte[] data) {
List<String> paths = getAllNecessaryPath(path);
for (int i = paths.size() - 1; i > 0; i--) {
String parent = paths.get(i);
PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
try {
s3.putObject(req);
} catch (Exception e) {
LogKit.error("[S3] Failed to create parent path {}", parent);
}
}
ObjectMetadata metadata;
try {
metadata = s3.getObjectMetadata(bucket, path);
@ -150,17 +146,17 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean createFile(String path) {
List<String> paths = getAllNecessaryPath(path);
for (int i = paths.size() - 1; i >= 0; i--) {
String parent = paths.get(i);
PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
String parent = path.substring(0, path.lastIndexOf(DELIMITER) + 1);
if (!dirExist(parent)) {
createDirectory(parent);
}
PutObjectRequest req = new PutObjectRequest(bucket, path, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
try {
s3.putObject(req);
} catch (Exception e) {
LogKit.error("[S3] Failed to create parent path {}", parent);
LogKit.error("[S3] Failed to create file path {}", path);
return false;
}
}
return true;
}
@ -170,17 +166,23 @@ public class S3ResourceRepository extends BaseResourceRepository {
if (!path.endsWith(DELIMITER)) {
path += DELIMITER;
}
List<String> paths = getAllNecessaryPath(path);
for (int i = paths.size() - 1; i >= 0; i--) {
String parent = paths.get(i);
PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
String temp = path.substring(0, path.length() - 1);
String parent = temp.substring(0, temp.lastIndexOf(DELIMITER) + 1);
if (StringUtils.isEmpty(parent)) {
return true;
}
if (!dirExist(parent)) {
if (!createDirectory(parent)) {
return false;
}
}
PutObjectRequest req = new PutObjectRequest(bucket, path, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata());
try {
s3.putObject(req);
} catch (Exception e) {
LogKit.error("[S3] Failed to create parent path {}", parent);
LogKit.error("[S3] Failed to create parent path {}", path);
return false;
}
}
return true;
}
@ -214,13 +216,49 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean exist(String path) {
return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path);
}
private boolean fileExist(String path) {
try {
return s3.doesObjectExist(bucket, path) || s3.doesObjectExist(bucket, path + DELIMITER);
return s3.doesObjectExist(bucket, path);
} catch (Exception e) {
return false;
}
}
private boolean dirExist(String path) {
if (StringUtils.equals(path, DELIMITER)) {
return true;
}
if (!path.endsWith(DELIMITER)) {
path += DELIMITER;
}
return fileExist(path);
}
/**
* 如果存在文件创建了，但是其父目录没有创建的场景，为其递归创建对象，返回创建结果
*/
private boolean isParentPathAbsent(String path) {
if (path.startsWith(DELIMITER)) {
path = path.substring(1);
}
if (path.endsWith(DELIMITER)) {
path = path.substring(0, path.length() - 1);
}
ObjectListing objectListing = s3.listObjects(
new ListObjectsRequest()
.withBucketName(bucket)
.withPrefix(path));
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
if (summary.getKey().startsWith(path + DELIMITER)) {
return createDirectory(path);
}
}
return false;
}
@Override
public String[] list(String dir, final Filter<String> filter) {
List<String> result = new ArrayList<>();

Write Preview
Loading…
Cancel
Save