From 6eaeaa81539021e6a0b5e16e7159f21ea0144d1f Mon Sep 17 00:00:00 2001 From: "Ocean.Hu" <2232761016@qq.com> Date: Sun, 7 May 2023 21:46:35 +0800 Subject: [PATCH 01/13] =?UTF-8?q?REPORT-95454=20bugfix=EF=BC=9Ahttp?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=A0=B9=E6=8D=AEendpoint=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fs/s3/repository/core/S3RepositoryFactory.java | 11 ++++++++--- .../fs/s3/repository/core/S3ResourceRepository.java | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java index 560d81e..2aecad2 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java @@ -13,6 +13,8 @@ import com.fr.io.context.info.RepositoryProfile; import com.fr.io.repository.ResourceRepository; import com.fr.stable.StringUtils; +import static com.fanruan.fs.s3.repository.core.S3ResourceRepository.HTTP; + /** * @author richie * @version 10.0 @@ -42,16 +44,19 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword()); AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion())) - .withCredentials(new AWSStaticCredentialsProvider(credentials)); + .withCredentials(new AWSStaticCredentialsProvider(credentials)).disableChunkedEncoding(); if (config.isEnablePathStyleAccess()) { amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); } + ClientConfiguration clientConfiguration = new ClientConfiguration(); if (StringUtils.isNotEmpty(config.getSignerOverride())) { - ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setSignerOverride(config.getSignerOverride()); + } + LogKit.debug("[S3] endpoint is {}", config.getEndPoint()); + if (config.getEndPoint().startsWith(HTTP)) { clientConfiguration.setProtocol(Protocol.HTTP); - amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); } + amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); AmazonS3 s3 = amazonS3ClientBuilder.build(); s3.listObjects(config.getBucket()); } catch (Exception e) { diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 66f907e..d96756e 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -37,6 +37,7 @@ import java.util.List; public class S3ResourceRepository extends BaseResourceRepository { private static final String DELIMITER = "/"; + public static final String HTTP = "http:"; private final AmazonS3 s3; private final String bucket; @@ -55,6 +56,10 @@ public class S3ResourceRepository extends BaseResourceRepository { if (StringUtils.isNotEmpty(config.getSignerOverride())) { clientConfiguration.setSignerOverride(config.getSignerOverride()); } + LogKit.debug("[S3] endpoint is {}", config.getEndPoint()); + if (config.getEndPoint().startsWith(HTTP)) { + clientConfiguration.setProtocol(Protocol.HTTP); + } clientConfiguration.setProtocol(Protocol.HTTP); amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); this.s3 = amazonS3ClientBuilder.build(); From 1313dda374af079fff451ed2dc0c60b7a5c9bfed Mon Sep 17 00:00:00 2001 From: "Ocean.Hu" <2232761016@qq.com> Date: Mon, 8 May 2023 16:13:00 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=E6=8F=90?= =?UTF-8?q?=E5=8D=87jartime=E5=92=8C=E7=89=88=E6=9C=AC=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin.xml b/plugin.xml index e664cbd..048adb2 100644 --- a/plugin.xml +++ b/plugin.xml @@ -5,9 +5,9 @@ com.fanruan.fs yes no - 1.3.7 + 1.3.8 10.0~10.0 - 2021-03-11 + 2023-03-14 richie Date: Tue, 23 May 2023 17:45:03 +0800 Subject: [PATCH 03/13] =?UTF-8?q?REPORT-96529=20fix:=20list=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=95=B0=E9=87=8F=E9=99=90=E5=88=B6;=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E6=8E=A5=E5=8F=A3=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3ResourceRepository.java | 113 ++++++++---------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index d96756e..5a21574 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -7,6 +7,7 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; @@ -36,6 +37,8 @@ import java.util.List; */ public class S3ResourceRepository extends BaseResourceRepository { + private static final int PAGE_SIZE = 1000; + private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -193,32 +196,49 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public boolean delete(String path) { - - s3.deleteObject(bucket, path); - String prefix = path; - if (!path.endsWith(DELIMITER)) { - prefix = path + DELIMITER; - } try { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) - .withPrefix(prefix).withDelimiter(DELIMITER); - ObjectListing objectListing = s3.listObjects(listObjectsRequest); - for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { - String key = summary.getKey(); - if (!key.endsWith(DELIMITER)) { - s3.deleteObject(bucket, key); + if (isDirectory(path)) { + if (!path.endsWith(DELIMITER)) { + path += DELIMITER; } - } - for (String pre : objectListing.getCommonPrefixes()) { - delete(pre); + deleteDirectory(path); + } else { + deleteFile(path); } } catch (Exception e) { - LogKit.error(e.getMessage(), e); + LogKit.error("[S3] delete {} failed, error message: {}", path, e.getMessage()); + return false; } - s3.deleteObject(bucket, prefix); return true; } + private void deleteFile(String path) throws Exception { + s3.deleteObject(bucket, path); + } + + private void deleteDirectory(String path) throws Exception { + List files = new ArrayList<>(); + for (FineFileEntry fineFileEntry : listEntry(path)) { + if (fineFileEntry.isDirectory()) { + deleteDirectory(fineFileEntry.getPath()); + } else { + files.add(fineFileEntry.getPath()); + } + } + deleteFiles(files); + deleteFile(path); + } + + private void deleteFiles(List paths) throws Exception { + //最多只能同时指定1000个key + for (int i = 0; i < paths.size(); i = i + PAGE_SIZE) { + int toIndex = Math.min(i + PAGE_SIZE, paths.size()); + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket) + .withKeys(paths.subList(i, toIndex).toArray(new String[0])); + s3.deleteObjects(deleteObjectsRequest); + } + } + @Override public boolean exist(String path) { return fileExist(path) || (!path.endsWith(DELIMITER) && dirExist(path)) || isParentPathAbsent(path); @@ -273,34 +293,28 @@ public class S3ResourceRepository extends BaseResourceRepository { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) .withPrefix(dir).withDelimiter(DELIMITER); ObjectListing objectListing = s3.listObjects(listObjectsRequest); - for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + collectFileName(dir, result, objectListing); + while (objectListing.isTruncated()) { + objectListing = s3.listNextBatchOfObjects(objectListing); + collectFileName(dir, result, objectListing); + } + if (filter != null) { + return result.stream().filter(filter::accept).toArray(String[]::new); + } + return result.toArray(new String[0]); + } + private void collectFileName(String dir, List result, ObjectListing objectListing) { + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String key = objectSummary.getKey(); if (StringKit.equals(key, dir)) { continue; } - String[] arr = key.split(DELIMITER); - String name = arr[arr.length - 1]; - if (filter == null) { - result.add(name); - } else { - if (filter.accept(name)) { - result.add(name); - } - } + result.add(key.substring(key.lastIndexOf(DELIMITER) + 1)); } for (String prefix : objectListing.getCommonPrefixes()) { - String[] arr = prefix.split(DELIMITER); - String name = arr[arr.length - 1] + DELIMITER; - if (filter == null) { - result.add(name); - } else { - if (filter.accept(name)) { - result.add(name); - } - } + result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1)); } - return result.toArray(new String[0]); } @Override @@ -406,27 +420,4 @@ public class S3ResourceRepository extends BaseResourceRepository { return entry; } - /** - * 递归创建父目录的metadata. 比如path是 {@code WEB-INF/reportlets/test/1.cpt} - * 则返回 {@code [WEB-INF/reportlets/test/1.cpt, WEB-INF/reportlets/test/, WEB-INF/reportlets/, WEB-INF/]} - */ - private List getAllNecessaryPath(String path) { - - // 获取所有path路径及其所有父路径 - List allPath = new ArrayList<>(); - allPath.add(path); - int lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - // 以/结尾的先把末尾的/去掉 - if (lastIdxOfDelimiter == path.length() - 1) { - path = path.substring(0, lastIdxOfDelimiter); - lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - } - while (lastIdxOfDelimiter > 0) { - allPath.add(path.substring(0, lastIdxOfDelimiter + 1)); - path = path.substring(0, lastIdxOfDelimiter); - lastIdxOfDelimiter = path.lastIndexOf(DELIMITER); - } - return allPath; - } - } From 590584026d4e054d0f72db8eaff37e0cedf668cd Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:13 +0800 Subject: [PATCH 04/13] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=B1=A0=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fanruan/fs/s3/repository/core/S3Config.java | 15 +++++++++++++++ .../s3/repository/core/S3ResourceRepository.java | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java index 5821864..976a9b8 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3Config.java @@ -38,6 +38,9 @@ public class S3Config extends CommonRepoConfig { @Identifier("signerOverride") private Conf signerOverride = HolderKit.simple(StringUtils.EMPTY); + @Identifier("maxConnections") + private Conf maxConnections = HolderKit.simple(200); + @GetConfig("endPoint") public String getEndPoint() { return endPoint.get(); @@ -98,6 +101,16 @@ public class S3Config extends CommonRepoConfig { this.signerOverride.set(signerOverride); } + @GetConfig("maxConnections") + public int getMaxConnections() { + return maxConnections.get(); + } + + @SetConfig("maxConnections") + public void setMaxConnections(int maxConnections) { + this.maxConnections.set(maxConnections); + } + @Override public void update(String key) { super.update(key); @@ -109,6 +122,7 @@ public class S3Config extends CommonRepoConfig { this.setBucket(newConfig.getBucket()); this.setEnablePathStyleAccess(newConfig.isEnablePathStyleAccess()); this.setSignerOverride(newConfig.getSignerOverride()); + this.setMaxConnections(newConfig.getMaxConnections()); } } @@ -121,6 +135,7 @@ public class S3Config extends CommonRepoConfig { cloned.bucket = (Conf) bucket.clone(); cloned.enablePathStyleAccess = (Conf) enablePathStyleAccess.clone(); cloned.signerOverride = (Conf) signerOverride.clone(); + cloned.maxConnections = (Conf) maxConnections.clone(); return cloned; } } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 5a21574..ca2b05d 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -15,6 +15,7 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; @@ -56,6 +57,8 @@ public class S3ResourceRepository extends BaseResourceRepository { amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); } ClientConfiguration clientConfiguration = new ClientConfiguration(); + clientConfiguration.setMaxConnections(config.getMaxConnections()); + LogKit.info("Max connections is {}!", clientConfiguration.getMaxConnections()); if (StringUtils.isNotEmpty(config.getSignerOverride())) { clientConfiguration.setSignerOverride(config.getSignerOverride()); } @@ -128,6 +131,8 @@ public class S3ResourceRepository extends BaseResourceRepository { try { return s3.getObject(request).getObjectContent(); } catch (Exception e) { + LogKit.error("[S3] Failed to read file {}", filePath); + LogKit.error(e.getMessage(), e); return new ByteArrayInputStream(new byte[0]); } } @@ -341,6 +346,8 @@ public class S3ResourceRepository extends BaseResourceRepository { S3Object s3Object = s3.getObject(bucket, path); if (s3Object != null) { try { + //s3Object要全部读完，否则会有警告 + IOUtils.copy(s3Object.getObjectContent(), new NullOutputStream()); return s3Object.getObjectMetadata().getLastModified().getTime(); } finally { s3Object.close(); From b1b2c507cb4e1138f90c66e93a4f57bdcd15695d Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 8 Jun 2023 11:31:36 +0800 Subject: [PATCH 05/13] =?UTF-8?q?REPORT-97765=20feat:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E6=B1=A0=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fanruan/fs/s3/repository/core/S3ResourceRepository.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index ca2b05d..614edec 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -22,6 +22,7 @@ import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; import com.fr.stable.Filter; import com.fr.stable.StringUtils; +import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream; From 7d4a1f54e3d93e324e13d68aadf3265a853db257 Mon Sep 17 00:00:00 2001 From: Afly Date: Thu, 15 Jun 2023 09:58:41 +0800 Subject: [PATCH 06/13] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20fix:=20?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=BF=9E=E6=8E=A5=E5=90=8E=E9=87=8A=E6=94=BE?= =?UTF-8?q?=E8=B5=84=E6=BA=90;list=E6=8E=A5=E5=8F=A3=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E8=87=AA=E8=BA=AB=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3RepositoryFactory.java | 7 ++++++- .../repository/core/S3ResourceRepository.java | 20 ++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java index 2aecad2..5de506c 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3RepositoryFactory.java @@ -40,6 +40,7 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { @Override public boolean verifyConfig(S3Config config) { + AmazonS3 s3 = null; try { BasicAWSCredentials credentials = new BasicAWSCredentials(config.getAccessKeyId(), config.getPassword()); AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() @@ -57,11 +58,15 @@ public class S3RepositoryFactory extends ConfigRepositoryFactory { clientConfiguration.setProtocol(Protocol.HTTP); } amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); - AmazonS3 s3 = amazonS3ClientBuilder.build(); + s3 = amazonS3ClientBuilder.build(); s3.listObjects(config.getBucket()); } catch (Exception e) { LogKit.error(e.getMessage(), e); return false; + } finally { + if (s3 != null) { + s3.shutdown(); + } } return true; } diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 614edec..21e5d7b 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -99,15 +99,15 @@ public class S3ResourceRepository extends BaseResourceRepository { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) .withPrefix(dir).withDelimiter(DELIMITER); ObjectListing objectListing = s3.listObjects(listObjectsRequest); - collectFileEntry(result, objectListing); + collectFileEntry(dir, result, objectListing); while (objectListing.isTruncated()) { objectListing = s3.listNextBatchOfObjects(objectListing); - collectFileEntry(result, objectListing); + collectFileEntry(dir, result, objectListing); } return result.toArray(new FineFileEntry[0]); } - private void collectFileEntry(List result, ObjectListing objectListing) { + private void collectFileEntry(String dir, List result, ObjectListing objectListing) { for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { String key = summary.getKey(); if (!key.endsWith(DELIMITER)) { @@ -115,9 +115,11 @@ public class S3ResourceRepository extends BaseResourceRepository { } } for (String prefix : objectListing.getCommonPrefixes()) { - FineFileEntry entry = new FineFileEntry(prefix); - entry.setDirectory(true); - result.add(entry); + if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) { + FineFileEntry entry = new FineFileEntry(prefix); + entry.setDirectory(true); + result.add(entry); + } } } @@ -319,7 +321,11 @@ public class S3ResourceRepository extends BaseResourceRepository { result.add(key.substring(key.lastIndexOf(DELIMITER) + 1)); } for (String prefix : objectListing.getCommonPrefixes()) { - result.add(prefix.substring(prefix.lastIndexOf(DELIMITER) + 1)); + if (StringUtils.isNotEmpty(prefix.substring(dir.length()).replaceAll(DELIMITER, StringUtils.EMPTY))) { + String[] arr = prefix.split(DELIMITER); + String name = arr[arr.length - 1] + DELIMITER; + result.add(name); + } } } From 649f712ab364afcb80db4170b522be330e2a2a2b Mon Sep 17 00:00:00 2001 From: Afly Date: Mon, 3 Jul 2023 14:24:53 +0800 Subject: [PATCH 07/13] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20chore:?= =?UTF-8?q?=20=E6=8F=90=E5=8D=87=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin.xml b/plugin.xml index 048adb2..0821aa9 100644 --- a/plugin.xml +++ b/plugin.xml @@ -5,12 +5,13 @@ com.fanruan.fs yes no - 1.3.8 + 1.3.9 10.0~10.0 2023-03-14 richie [2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。
[2022-09-22]第三方组件升级。
From 67c37b888c4a8c03b8aaa0f6507fae2d5fbfe40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aeolus=2EZhang-=E5=BC=A0=E6=95=AC=E5=B3=A5?= Date: Mon, 3 Jul 2023 15:47:27 +0800 Subject: [PATCH 08/13] =?UTF-8?q?REPORT-99648=20fix:s3=20fineui=E5=85=BC?= =?UTF-8?q?=E5=AE=B9=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +- front/bundle.js | 318 ++++++++++++++++++ front/gulpfile.js | 10 + front/package.json | 13 + .../fanruan/fs/s3/repository/web/js/bundle.js | 312 +---------------- 5 files changed, 345 insertions(+), 312 deletions(-) create mode 100644 front/bundle.js create mode 100644 front/gulpfile.js create mode 100644 front/package.json diff --git a/.gitignore b/.gitignore index 29c9cac..256375a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ target/ build local.properties classes/ -transform-classes/ \ No newline at end of file +transform-classes/ +node_modules +yarn.lock \ No newline at end of file diff --git a/front/bundle.js b/front/bundle.js new file mode 100644 index 0000000..bdfc9ed --- /dev/null +++ b/front/bundle.js @@ -0,0 +1,318 @@ +BI.config("dec.constant.intelligence.cluster.file.server", function (items) { + items.push({ + value: "S3", // 地址栏显示的hash值 + id: "decision-intelligence-cluster-file-s3", // id + text: "S3", // 文字 + cardType: "dec.intelligence.cluster.file.s3", + workRoot: false, + }); + + return items; +}); + + +!(function () { + var LABEL_WIDTH = 107, EDITOR_WIDTH = 393; + var S3 = BI.inherit(BI.Widget, { + + props: { + baseCls: "dec-cluster-ftp", + value: {}, + }, + + _store: function () { + return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", { + value: this.options.value, + }); + }, + + watch: { + isOpen: function (v) { + this.toggle.setVisible(v); + } + }, + + render: function () { + var self = this, o = this.options; + + return { + type: "bi.vertical", + tgap: 15, + items: [ + { + type: "dec.label.editor.item", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Input"), + text: BI.i18nText("Plugin-S3_End_Point"), + value: this.model.endPoint, + el: { + disabled: !o.editable, + }, + ref: function (_ref) { + self.endPointRow = _ref; + }, + listeners: [{ + eventName: BI.Editor.EVENT_CHANGE, + action: function () { + self.store.setEndPoint(this.getValue()); + }, + }], + }, { + type: "dec.label.editor.item", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Input"), + text: BI.i18nText("Plugin-S3_Region"), + value: this.model.region, + el: { + disabled: !o.editable, + }, + ref: function (_ref) { + self.regionRow = _ref; + }, + listeners: [{ + eventName: BI.Editor.EVENT_CHANGE, + action: function () { + self.store.setRegion(this.getValue()); + }, + }], + }, { + type: "dec.label.editor.item", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Input"), + text: BI.i18nText("Plugin-S3_Access_Key_Id"), + value: this.model.accessKeyId, + el: { + disabled: !o.editable, + }, + ref: function (_ref) { + self.portRow = _ref; + }, + listeners: [{ + eventName: BI.Editor.EVENT_CHANGE, + action: function () { + self.store.setAccessKeyId(this.getValue()); + }, + }], + }, + { + type: "dec.common.cipher.editor", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"), + text: BI.i18nText("Plugin-S3_Access_Key_Secret"), + value: this.model.password, + el: { + disabled: !o.editable, + }, + ref: function (_ref) { + self.passwordRow = _ref; + }, + }, + { + type: "dec.label.editor.item", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Bucket"), + text: BI.i18nText("Plugin-S3_Bucket"), + value: this.model.bucket, + el: { + disabled: !o.editable, + }, + listeners: [{ + eventName: BI.Editor.EVENT_CHANGE, + action: function () { + self.store.setBucket(this.getValue()); + }, + }], + }, + { + type: "dec.label.editor.item", + el: { + disabled: !o.editable, + }, + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Dec-Please_Input"), + text: BI.i18nText("Dec-Basic_Path"), + value: this.model.workRoot, + ref: function (_ref) { + self.filePathRow = _ref; + }, + }, { + type: "bi.vertical_adapt", + items: [{ + type: "bi.icon_change_button", + iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font", + ref: (_ref) => { + this.OtherConfigButton = _ref; + }, + handler: () => { + this.store.setIsOpen(!this.model.isOpen); + this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font"); + } + }, { + type: "bi.text_button", + text: BI.i18nText('Plugin-S3_Other_Config'), + handler: () => { + this.store.setIsOpen(!this.model.isOpen); + this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font"); + } + }] + }, { + type: 'bi.vertical', + ref: (_ref) => { + this.toggle = _ref; + }, + invisible: () => !this.model.isOpen, + items: [{ + type: "dec.label.editor.item", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Input"), + text: "PathStyleAccess", + value: this.model.enablePathStyleAccess, + ref: function (_ref) { + self.enablePathStyleAccessRow = _ref; + }, + el: { + disabled: !o.editable, + }, + listeners: [{ + eventName: BI.Editor.EVENT_CHANGE, + action: function () { + self.store.setEnablePathStyleAccess(this.getValue()); + } + }] + }, { + type: "dec.label.editor.item", + textWidth: LABEL_WIDTH, + editorWidth: EDITOR_WIDTH, + watermark: BI.i18nText("Plugin-S3_Input"), + text: "SignerOverride", + value: this.model.signerOverride, + el: { + disabled: !o.editable, + }, + tgap: 15, + listeners: [{ + eventName: BI.Editor.EVENT_CHANGE, + action: function () { + self.store.setSignerOverride(this.getValue()); + } + }] + }], + } + ] + }; + }, + + getValue: function () { + var enablePathStyleAccess = false; + if (this.model.enablePathStyleAccess === 'true') { + enablePathStyleAccess = true; + } + + return { + endPoint: this.model.endPoint, + region: this.model.region, + accessKeyId: this.model.accessKeyId, + password: this.passwordRow.getCipher(), + bucket: this.model.bucket, + workRoot: this.filePathRow.getValue(), + enablePathStyleAccess, + signerOverride: this.model.signerOverride, + }; + }, + + validation: function () { + var valid = true; + var path = this.filePathRow.getValue(); + if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) { + this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")); + valid = false; + } + if (BI.startWith(path, "/") || !BI.endWith(path, "/")) { + this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")); + valid = false; + } + if (!BI.isKey(path)) { + this.filePathRow.showError(BI.i18nText("Dec-Error_Null")); + valid = false; + } + if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) { + this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")); + valid = false; + } + + return valid; + }, + + }); + BI.shortcut("dec.intelligence.cluster.file.s3", S3); +}()); + + +!(function () { + var Model = BI.inherit(Fix.Model, { + + state: function () { + var val = this.options.value; + + return { + endPoint: val.endPoint, + region: val.region, + accessKeyId: val.accessKeyId, + password: val.password, + bucket: val.bucket, + workRoot: val.workRoot, + isOpen: false, + enablePathStyleAccess: 'false', + signerOverride: "", + }; + }, + + computed: { + encodingArray: function () { + return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) { + return { + value: v, + }; + }); + }, + }, + + actions: { + setEndPoint: function (v) { + this.model.endPoint = v; + }, + + setRegion: function (v) { + this.model.region = v; + }, + + setAccessKeyId: function (v) { + this.model.accessKeyId = v; + }, + + setBucket: function (v) { + this.model.bucket = v; + }, + + setEnablePathStyleAccess: function (v) { + this.model.enablePathStyleAccess = v; + }, + + setSignerOverride: function (v) { + this.model.signerOverride = v; + }, + + setIsOpen: function (v) { + this.model.isOpen = v; + }, + }, + }); + BI.model("dec.model.intelligence.cluster.file.s3", Model); +}()); diff --git a/front/gulpfile.js b/front/gulpfile.js new file mode 100644 index 0000000..bc6eefa --- /dev/null +++ b/front/gulpfile.js @@ -0,0 +1,10 @@ +const gulp = require('gulp'); +const uglify = require('gulp-uglify'); + +function build() { + return gulp.src('./bundle.js') + .pipe(uglify()) + .pipe(gulp.dest('../src/main/resources/com/fanruan/fs/s3/repository/web/js')); +} + +exports.default = build; \ No newline at end of file diff --git a/front/package.json b/front/package.json new file mode 100644 index 0000000..10e2932 --- /dev/null +++ b/front/package.json @@ -0,0 +1,13 @@ +{ + "name": "front", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "scripts": { + "build": "gulp" + }, + "dependencies": { + "gulp": "^4.0.2", + "gulp-uglify": "^3.0.2" + } +} diff --git a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js index af8b1bd..b86e382 100644 --- a/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js +++ b/src/main/resources/com/fanruan/fs/s3/repository/web/js/bundle.js @@ -1,311 +1 @@ -BI.config("dec.constant.intelligence.cluster.file.server", function (items) { - items.push({ - value: "S3", // 地址栏显示的hash值 - id: "decision-intelligence-cluster-file-s3", // id - text: "S3", // 文字 - cardType: "dec.intelligence.cluster.file.s3", - workRoot: false, - }); - - return items; -}); - - -!(function () { - var LABEL_WIDTH = 107, EDITOR_WIDTH = 393; - var S3 = BI.inherit(BI.Widget, { - - props: { - baseCls: "dec-cluster-ftp", - value: {}, - }, - - _store: function () { - return BI.Models.getModel("dec.model.intelligence.cluster.file.s3", { - value: this.options.value, - }); - }, - - watch: {}, - - render: function () { - var self = this, o = this.options; - - return { - type: "bi.vertical", - tgap: 15, - items: [ - { - type: "dec.label.editor.item", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Input"), - text: BI.i18nText("Plugin-S3_End_Point"), - value: this.model.endPoint, - el: { - disabled: !o.editable, - }, - ref: function (_ref) { - self.endPointRow = _ref; - }, - listeners: [{ - eventName: BI.Editor.EVENT_CHANGE, - action: function () { - self.store.setEndPoint(this.getValue()); - }, - }], - }, { - type: "dec.label.editor.item", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Input"), - text: BI.i18nText("Plugin-S3_Region"), - value: this.model.region, - el: { - disabled: !o.editable, - }, - ref: function (_ref) { - self.regionRow = _ref; - }, - listeners: [{ - eventName: BI.Editor.EVENT_CHANGE, - action: function () { - self.store.setRegion(this.getValue()); - }, - }], - }, { - type: "dec.label.editor.item", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Input"), - text: BI.i18nText("Plugin-S3_Access_Key_Id"), - value: this.model.accessKeyId, - el: { - disabled: !o.editable, - }, - ref: function (_ref) { - self.portRow = _ref; - }, - listeners: [{ - eventName: BI.Editor.EVENT_CHANGE, - action: function () { - self.store.setAccessKeyId(this.getValue()); - }, - }], - }, - { - type: "dec.common.cipher.editor", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Access_Key_Secret"), - text: BI.i18nText("Plugin-S3_Access_Key_Secret"), - value: this.model.password, - el: { - disabled: !o.editable, - }, - ref: function (_ref) { - self.passwordRow = _ref; - }, - }, - { - type: "dec.label.editor.item", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Bucket"), - text: BI.i18nText("Plugin-S3_Bucket"), - value: this.model.bucket, - el: { - disabled: !o.editable, - }, - listeners: [{ - eventName: BI.Editor.EVENT_CHANGE, - action: function () { - self.store.setBucket(this.getValue()); - }, - }], - }, - { - type: "dec.label.editor.item", - el: { - disabled: !o.editable, - }, - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Dec-Please_Input"), - text: BI.i18nText("Dec-Basic_Path"), - value: this.model.workRoot, - ref: function (_ref) { - self.filePathRow = _ref; - }, - }, { - type: "bi.vertical_adapt", - items: [{ - type: "bi.icon_change_button", - iconCls: this.model.isOpen ? "expander-down-font" : "expander-right-font", - ref: (_ref) => { - this.OtherConfigButton = _ref; - }, - handler: () => { - this.store.setIsOpen(!this.model.isOpen); - this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font"); - } - }, { - type: "bi.text_button", - text: BI.i18nText('Plugin-S3_Other_Config'), - handler: () => { - this.store.setIsOpen(!this.model.isOpen); - this.OtherConfigButton.setIcon(this.model.isOpen ? "expander-down-font" : "expander-right-font"); - } - }] - }, { - type: 'bi.vertical', - invisible: () => !this.model.isOpen, - items: [{ - type: "dec.label.editor.item", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Input"), - text: "PathStyleAccess", - value: this.model.enablePathStyleAccess, - ref: function (_ref) { - self.enablePathStyleAccessRow = _ref; - }, - el: { - disabled: !o.editable, - }, - listeners: [{ - eventName: BI.Editor.EVENT_CHANGE, - action: function () { - self.store.setEnablePathStyleAccess(this.getValue()); - } - }] - }, { - type: "dec.label.editor.item", - textWidth: LABEL_WIDTH, - editorWidth: EDITOR_WIDTH, - watermark: BI.i18nText("Plugin-S3_Input"), - text: "SignerOverride", - value: this.model.signerOverride, - el: { - disabled: !o.editable, - }, - tgap: 15, - listeners: [{ - eventName: BI.Editor.EVENT_CHANGE, - action: function () { - self.store.setSignerOverride(this.getValue()); - } - }] - }], - } - ] - }; - }, - - getValue: function () { - var enablePathStyleAccess = false; - if (this.model.enablePathStyleAccess === 'true') { - enablePathStyleAccess = true; - } - - return { - endPoint: this.model.endPoint, - region: this.model.region, - accessKeyId: this.model.accessKeyId, - password: this.passwordRow.getCipher(), - bucket: this.model.bucket, - workRoot: this.filePathRow.getValue(), - enablePathStyleAccess, - signerOverride: this.model.signerOverride, - }; - }, - - validation: function () { - var valid = true; - var path = this.filePathRow.getValue(); - if (Dec.Utils.strLength(path) > DecCst.STRING_SHORT_TEXT_LENGTH) { - this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")); - valid = false; - } - if (BI.startWith(path, "/") || !BI.endWith(path, "/")) { - this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")); - valid = false; - } - if (!BI.isKey(path)) { - this.filePathRow.showError(BI.i18nText("Dec-Error_Null")); - valid = false; - } - if (!(this.model.enablePathStyleAccess === 'false' || this.model.enablePathStyleAccess === 'true')) { - this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")); - valid = false; - } - - return valid; - }, - - }); - BI.shortcut("dec.intelligence.cluster.file.s3", S3); -}()); - - -!(function () { - var Model = BI.inherit(Fix.Model, { - - state: function () { - var val = this.options.value; - - return { - endPoint: val.endPoint, - region: val.region, - accessKeyId: val.accessKeyId, - password: val.password, - bucket: val.bucket, - workRoot: val.workRoot, - isOpen: false, - enablePathStyleAccess: 'false', - signerOverride: "", - }; - }, - - computed: { - encodingArray: function () { - return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY, function (i, v) { - return { - value: v, - }; - }); - }, - }, - - actions: { - setEndPoint: function (v) { - this.model.endPoint = v; - }, - - setRegion: function (v) { - this.model.region = v; - }, - - setAccessKeyId: function (v) { - this.model.accessKeyId = v; - }, - - setBucket: function (v) { - this.model.bucket = v; - }, - - setEnablePathStyleAccess: function (v) { - this.model.enablePathStyleAccess = v; - }, - - setSignerOverride: function (v) { - this.model.signerOverride = v; - }, - - setIsOpen: function (v) { - this.model.isOpen = v; - }, - }, - }); - BI.model("dec.model.intelligence.cluster.file.s3", Model); -}()); +BI.config("dec.constant.intelligence.cluster.file.server",function(e){return e.push({value:"S3",id:"decision-intelligence-cluster-file-s3",text:"S3",cardType:"dec.intelligence.cluster.file.s3",workRoot:!1}),e}),function(){var i=107,n=393,e=BI.inherit(BI.Widget,{props:{baseCls:"dec-cluster-ftp",value:{}},_store:function(){return BI.Models.getModel("dec.model.intelligence.cluster.file.s3",{value:this.options.value})},watch:{isOpen:function(e){this.toggle.setVisible(e)}},render:function(){var t=this,e=this.options;return{type:"bi.vertical",tgap:15,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_End_Point"),value:this.model.endPoint,el:{disabled:!e.editable},ref:function(e){t.endPointRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEndPoint(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Region"),value:this.model.region,el:{disabled:!e.editable},ref:function(e){t.regionRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setRegion(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:BI.i18nText("Plugin-S3_Access_Key_Id"),value:this.model.accessKeyId,el:{disabled:!e.editable},ref:function(e){t.portRow=e},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setAccessKeyId(this.getValue())}}]},{type:"dec.common.cipher.editor",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Access_Key_Secret"),text:BI.i18nText("Plugin-S3_Access_Key_Secret"),value:this.model.password,el:{disabled:!e.editable},ref:function(e){t.passwordRow=e}},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Bucket"),text:BI.i18nText("Plugin-S3_Bucket"),value:this.model.bucket,el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setBucket(this.getValue())}}]},{type:"dec.label.editor.item",el:{disabled:!e.editable},textWidth:i,editorWidth:n,watermark:BI.i18nText("Dec-Please_Input"),text:BI.i18nText("Dec-Basic_Path"),value:this.model.workRoot,ref:function(e){t.filePathRow=e}},{type:"bi.vertical_adapt",items:[{type:"bi.icon_change_button",iconCls:this.model.isOpen?"expander-down-font":"expander-right-font",ref:e=>{this.OtherConfigButton=e},handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}},{type:"bi.text_button",text:BI.i18nText("Plugin-S3_Other_Config"),handler:()=>{this.store.setIsOpen(!this.model.isOpen),this.OtherConfigButton.setIcon(this.model.isOpen?"expander-down-font":"expander-right-font")}}]},{type:"bi.vertical",ref:e=>{this.toggle=e},invisible:()=>!this.model.isOpen,items:[{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"PathStyleAccess",value:this.model.enablePathStyleAccess,ref:function(e){t.enablePathStyleAccessRow=e},el:{disabled:!e.editable},listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setEnablePathStyleAccess(this.getValue())}}]},{type:"dec.label.editor.item",textWidth:i,editorWidth:n,watermark:BI.i18nText("Plugin-S3_Input"),text:"SignerOverride",value:this.model.signerOverride,el:{disabled:!e.editable},tgap:15,listeners:[{eventName:BI.Editor.EVENT_CHANGE,action:function(){t.store.setSignerOverride(this.getValue())}}]}]}]}},getValue:function(){var e=!1;return"true"===this.model.enablePathStyleAccess&&(e=!0),{endPoint:this.model.endPoint,region:this.model.region,accessKeyId:this.model.accessKeyId,password:this.passwordRow.getCipher(),bucket:this.model.bucket,workRoot:this.filePathRow.getValue(),enablePathStyleAccess:e,signerOverride:this.model.signerOverride}},validation:function(){var e=!0,t=this.filePathRow.getValue();return Dec.Utils.strLength(t)>DecCst.STRING_SHORT_TEXT_LENGTH&&(this.filePathRow.showError(BI.i18nText("Dec-Error_Length_Greater_Than_Short_Text")),e=!1),!BI.startWith(t,"/")&&BI.endWith(t,"/")||(this.filePathRow.showError(BI.i18nText("Dec-Error_Start_With_Slash_Or_End_Without_Slash")),e=!1),BI.isKey(t)||(this.filePathRow.showError(BI.i18nText("Dec-Error_Null")),e=!1),"false"!==this.model.enablePathStyleAccess&&"true"!==this.model.enablePathStyleAccess&&(this.enablePathStyleAccessRow.showError(BI.i18nText("Plugin-S3_EnablePathStyleAccess_Error_Tip")),e=!1),e}});BI.shortcut("dec.intelligence.cluster.file.s3",e)}(),function(){var e=BI.inherit(Fix.Model,{state:function(){var e=this.options.value;return{endPoint:e.endPoint,region:e.region,accessKeyId:e.accessKeyId,password:e.password,bucket:e.bucket,workRoot:e.workRoot,isOpen:!1,enablePathStyleAccess:"false",signerOverride:""}},computed:{encodingArray:function(){return BI.map(DecCst.EncodeConstants.ENCODING_ARRAY,function(e,t){return{value:t}})}},actions:{setEndPoint:function(e){this.model.endPoint=e},setRegion:function(e){this.model.region=e},setAccessKeyId:function(e){this.model.accessKeyId=e},setBucket:function(e){this.model.bucket=e},setEnablePathStyleAccess:function(e){this.model.enablePathStyleAccess=e},setSignerOverride:function(e){this.model.signerOverride=e},setIsOpen:function(e){this.model.isOpen=e}}});BI.model("dec.model.intelligence.cluster.file.s3",e)}(); \ No newline at end of file From 13d945172ffdd31df591e9fd3ec2e3c24ab7c9c9 Mon Sep 17 00:00:00 2001 From: Afly Date: Mon, 24 Jul 2023 20:24:42 +0800 Subject: [PATCH 09/13] =?UTF-8?q?REPORT-100817=20fix:=20s3=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=AF=B9=E4=BA=8E=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3ResourceRepository.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 21e5d7b..e4b0a2e 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -7,25 +7,34 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; import com.amazonaws.util.IOUtils; import com.fanruan.api.log.LogKit; import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; +import com.fr.io.utils.ResourceIOUtils; +import com.fr.log.FineLoggerFactory; import com.fr.stable.Filter; import com.fr.stable.StringUtils; import com.fr.third.org.apache.commons.io.output.NullOutputStream; import com.fr.workspace.resource.ResourceIOException; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; @@ -41,6 +50,10 @@ public class S3ResourceRepository extends BaseResourceRepository { private static final int PAGE_SIZE = 1000; + private static final int PART_SIZE = 5 * 1024 * 1024; + + private static final int MULTIPART_UPLOAD_LIMIT = 4 * PART_SIZE; + private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -158,6 +171,61 @@ public class S3ResourceRepository extends BaseResourceRepository { s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); } + @Override + public void write(String path, InputStream inputStream) throws ResourceIOException { + long dataLength = 0; + try { + dataLength = inputStream.available(); + } catch (IOException e) { + FineLoggerFactory.getLogger().error(e.getMessage(), e); + } + //超过一定大小才使用分片上传，小文件来说，网络传输时间可能较短，且上传失败的风险相对较低。 + //在网络稳定的情况下，使用分片上传可能没有太多的优势，反而增加了额外开销和复杂性 + if (dataLength > MULTIPART_UPLOAD_LIMIT) { + try { + // Step 1: 初始化分片上传 + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path); + InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + + // Step 2: 分片上传文件 + List partETags = new ArrayList<>(); + long position = 0; + + for (int partNumber = 1; position < dataLength; partNumber++) { + // 最后一个分片可能小于5MB + long partSizeBytes = Math.min(PART_SIZE, dataLength - position); + byte[] bytes = new byte[(int) partSizeBytes]; + inputStream.read(bytes); + + // 创建上传请求 + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucket) + .withKey(path) + .withUploadId(uploadId) + .withPartNumber(partNumber) + .withInputStream(new ByteArrayInputStream(bytes)) + .withPartSize(partSizeBytes); + + // 上传分片 + UploadPartResult uploadResult = s3.uploadPart(uploadRequest); + partETags.add(uploadResult.getPartETag()); + + position += partSizeBytes; + } + + // Step 3: 完成分片上传 + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags); + s3.completeMultipartUpload(compRequest); + } catch (IOException e) { + throw new ResourceIOException(e); + } finally { + ResourceIOUtils.close(inputStream); + } + } else { + super.write(path, inputStream); + } + } @Override public boolean createFile(String path) { From 66beccd286fd44257f38be2620302227a0ef5d1a Mon Sep 17 00:00:00 2001 From: Afly Date: Mon, 7 Aug 2023 13:55:30 +0800 Subject: [PATCH 10/13] =?UTF-8?q?REPORT-102146=20fix:=20isDirectory?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E6=9F=90=E4=BA=9B=E5=9C=BA=E6=99=AF=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E4=B8=8D=E6=AD=A3=E7=A1=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3ResourceRepository.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index e4b0a2e..6576e5b 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -399,19 +399,16 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public boolean isDirectory(String path) { - - if (path.endsWith(DELIMITER) && exist(path)) { - return true; - } - ObjectListing listing = s3.listObjects(bucket, path); - if (listing.getObjectSummaries().isEmpty()) { - return false; - } - if (listing.getObjectSummaries().size() > 1) { - return true; + if (path.endsWith(DELIMITER)) { + return exist(path); } else { - S3ObjectSummary summary = listing.getObjectSummaries().get(0); - return !StringKit.equals(listing.getPrefix(), summary.getKey()); + ObjectListing listing = s3.listObjects(bucket, path); + List objectSummaries = listing.getObjectSummaries(); + if (objectSummaries.isEmpty()) { + return false; + } + String dirFormat = path + DELIMITER; + return objectSummaries.stream().anyMatch(s3ObjectSummary -> StringUtils.equals(s3ObjectSummary.getKey(), dirFormat)); } } From d762e6348310dc3c35e4e4cad90223ae143249e3 Mon Sep 17 00:00:00 2001 From: Afly Date: Mon, 7 Aug 2023 14:02:20 +0800 Subject: [PATCH 11/13] =?UTF-8?q?JSY-28624=20fix:=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E4=B8=8B=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/core/S3ResourceRepository.java | 117 ++++++++---------- 1 file changed, 55 insertions(+), 62 deletions(-) diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 6576e5b..105cccb 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -27,7 +27,6 @@ import com.fanruan.api.util.StringKit; import com.fr.io.repository.FineFileEntry; import com.fr.io.repository.base.BaseResourceRepository; import com.fr.io.utils.ResourceIOUtils; -import com.fr.log.FineLoggerFactory; import com.fr.stable.Filter; import com.fr.stable.StringUtils; import com.fr.third.org.apache.commons.io.output.NullOutputStream; @@ -52,7 +51,7 @@ public class S3ResourceRepository extends BaseResourceRepository { private static final int PART_SIZE = 5 * 1024 * 1024; - private static final int MULTIPART_UPLOAD_LIMIT = 4 * PART_SIZE; + private static final int MULTIPART_UPLOAD_LIMIT = 20 * PART_SIZE; private static final String DELIMITER = "/"; public static final String HTTP = "http:"; @@ -155,75 +154,69 @@ public class S3ResourceRepository extends BaseResourceRepository { @Override public void write(String path, byte[] data) { - ObjectMetadata metadata; - try { - metadata = s3.getObjectMetadata(bucket, path); - } catch (Exception e) { - metadata = new ObjectMetadata(); - String mimeType = URLConnection.guessContentTypeFromName(path); - if (mimeType != null) { - metadata.setContentType(mimeType); + int length = data.length; + if (length > MULTIPART_UPLOAD_LIMIT) { + multipartUpload(path, new ByteArrayInputStream(data)); + } else { + ObjectMetadata metadata; + try { + metadata = s3.getObjectMetadata(bucket, path); + } catch (Exception e) { + metadata = new ObjectMetadata(); + String mimeType = URLConnection.guessContentTypeFromName(path); + if (mimeType != null) { + metadata.setContentType(mimeType); + } } + if (metadata != null) { + metadata.setContentLength(length); + } + s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); } - if (metadata != null) { - metadata.setContentLength(data.length); - } - s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); } @Override public void write(String path, InputStream inputStream) throws ResourceIOException { - long dataLength = 0; + multipartUpload(path, inputStream); + } + + private void multipartUpload(String path, InputStream inputStream) { try { - dataLength = inputStream.available(); - } catch (IOException e) { - FineLoggerFactory.getLogger().error(e.getMessage(), e); - } - //超过一定大小才使用分片上传，小文件来说，网络传输时间可能较短，且上传失败的风险相对较低。 - //在网络稳定的情况下，使用分片上传可能没有太多的优势，反而增加了额外开销和复杂性 - if (dataLength > MULTIPART_UPLOAD_LIMIT) { - try { - // Step 1: 初始化分片上传 - InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path); - InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest); - String uploadId = initResponse.getUploadId(); - - // Step 2: 分片上传文件 - List partETags = new ArrayList<>(); - long position = 0; - - for (int partNumber = 1; position < dataLength; partNumber++) { - // 最后一个分片可能小于5MB - long partSizeBytes = Math.min(PART_SIZE, dataLength - position); - byte[] bytes = new byte[(int) partSizeBytes]; - inputStream.read(bytes); - - // 创建上传请求 - UploadPartRequest uploadRequest = new UploadPartRequest() - .withBucketName(bucket) - .withKey(path) - .withUploadId(uploadId) - .withPartNumber(partNumber) - .withInputStream(new ByteArrayInputStream(bytes)) - .withPartSize(partSizeBytes); - - // 上传分片 - UploadPartResult uploadResult = s3.uploadPart(uploadRequest); - partETags.add(uploadResult.getPartETag()); - - position += partSizeBytes; - } + // Step 1: 初始化分片上传 + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, path); + InitiateMultipartUploadResult initResponse = s3.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + + // Step 2: 分片上传文件 + List partETags = new ArrayList<>(); + byte[] buffer = new byte[PART_SIZE]; + int bytesRead; + int partNumber = 1; + + while ((bytesRead = inputStream.read(buffer)) > 0) { + // 创建上传请求 + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(bucket) + .withKey(path) + .withUploadId(uploadId) + .withPartNumber(partNumber) + .withInputStream(new ByteArrayInputStream(buffer, 0, bytesRead)) + .withPartSize(bytesRead); - // Step 3: 完成分片上传 - CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags); - s3.completeMultipartUpload(compRequest); - } catch (IOException e) { - throw new ResourceIOException(e); - } finally { - ResourceIOUtils.close(inputStream); + // 上传分片 + UploadPartResult uploadResult = s3.uploadPart(uploadRequest); + partETags.add(uploadResult.getPartETag()); + + partNumber++; } - } else { - super.write(path, inputStream); + + // Step 3: 完成分片上传 + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucket, path, uploadId, partETags); + s3.completeMultipartUpload(compRequest); + } catch (IOException e) { + throw new ResourceIOException(e); + } finally { + ResourceIOUtils.close(inputStream); } } From 3232e03cbdf1a56e57cdae46515c5c78eaf6cfb2 Mon Sep 17 00:00:00 2001 From: Afly Date: Tue, 8 Aug 2023 10:51:17 +0800 Subject: [PATCH 12/13] =?UTF-8?q?=E6=97=A0jira=E4=BB=BB=E5=8A=A1=20fix:=20?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E6=8F=92=E4=BB=B6=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 3 ++- .../fanruan/fs/s3/repository/core/S3ResourceRepository.java | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin.xml b/plugin.xml index 0821aa9..9883cd0 100644 --- a/plugin.xml +++ b/plugin.xml @@ -5,12 +5,13 @@ com.fanruan.fs yes no - 1.3.9 + 1.4.0 10.0~10.0 2023-03-14 richie [2023-06-30]修复默认配置获取错误的问题，过滤有问题的路径。
[2023-03-28]第三方组件升级。
[2023-01-03]优化写文件性能; 修复文件太多显示不全的问题。
diff --git a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java index 105cccb..f82d7ad 100644 --- a/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java +++ b/src/main/java/com/fanruan/fs/s3/repository/core/S3ResourceRepository.java @@ -79,7 +79,6 @@ public class S3ResourceRepository extends BaseResourceRepository { if (config.getEndPoint().startsWith(HTTP)) { clientConfiguration.setProtocol(Protocol.HTTP); } - clientConfiguration.setProtocol(Protocol.HTTP); amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); this.s3 = amazonS3ClientBuilder.build(); this.bucket = config.getBucket(); From 42d6a685f039b9ace8c4e6b6fa6686c4b126b3f9 Mon Sep 17 00:00:00 2001 From: "Ocean.Hu" <2232761016@qq.com> Date: Mon, 28 Aug 2023 10:02:21 +0800 Subject: [PATCH 13/13] =?UTF-8?q?=E6=97=A0jira,=E5=9B=9E=E9=80=80=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E6=8F=90=E5=8D=87=E7=9A=84jartime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugin.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.xml b/plugin.xml index 9883cd0..030424b 100644 --- a/plugin.xml +++ b/plugin.xml @@ -7,7 +7,7 @@ no 1.4.0 10.0~10.0 - 2023-03-14 + 2021-03-11 richie