From 43d7b3597769c40a5d8631a8aca1fdf08578d36e Mon Sep 17 00:00:00 2001 From: Zzih <466315465@qq.com> Date: Wed, 24 May 2023 05:43:44 +0000 Subject: [PATCH] [fix-14119] The checkBucketNameExists method should not use s3Client. listBuckets() (#14188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * optimize checkBucketNameExists * fix code style --------- Co-authored-by: 詹子恒 --- .../common/log/remote/S3RemoteLogHandler.java | 19 +++++++------------ .../plugin/storage/s3/S3StorageOperator.java | 19 +++++++------------ 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java index 00c632ecda..d1c8c41445 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/S3RemoteLogHandler.java @@ -35,7 +35,6 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; @@ -162,17 +161,13 @@ public class S3RemoteLogHandler implements RemoteLogHandler, Closeable { throw new IllegalArgumentException("remote.logging.s3.bucket.name is blank"); } - Bucket existsBucket = s3Client.listBuckets() - .stream() - .filter( - bucket -> bucket.getName().equals(bucketName)) - .findFirst() - .orElseThrow(() -> { - return new IllegalArgumentException( - "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); - }); - - log.info("bucketName: {} has been found, the current regionName is {}", existsBucket.getName(), + boolean existsBucket = s3Client.doesBucketExistV2(bucketName); + if (!existsBucket) { + throw new IllegalArgumentException( + "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); + } + + log.info("bucketName: {} has been found, the current regionName is {}", bucketName, s3Client.getRegionName()); } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index 781119c237..3b8aa31152 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -63,7 +63,6 @@ import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; @@ -404,17 +403,13 @@ public class S3StorageOperator implements Closeable, StorageOperate { throw new IllegalArgumentException("resource.aws.s3.bucket.name is blank"); } - Bucket existsBucket = s3Client.listBuckets() - .stream() - .filter( - bucket -> bucket.getName().equals(bucketName)) - .findFirst() - .orElseThrow(() -> { - return new IllegalArgumentException( - "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); - }); - - log.info("bucketName: {} has been found, the current regionName is {}", existsBucket.getName(), + boolean existsBucket = s3Client.doesBucketExistV2(bucketName); + if (!existsBucket) { + throw new IllegalArgumentException( + "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); + } + + log.info("bucketName: {} has been found, the current regionName is {}", bucketName, s3Client.getRegionName()); }