|
|
|
@ -18,8 +18,8 @@ import com.fanruan.api.log.LogKit;
|
|
|
|
|
import com.fanruan.api.util.StringKit; |
|
|
|
|
import com.fr.io.repository.FineFileEntry; |
|
|
|
|
import com.fr.io.repository.base.BaseResourceRepository; |
|
|
|
|
import com.fr.log.FineLoggerFactory; |
|
|
|
|
import com.fr.stable.Filter; |
|
|
|
|
import com.fr.stable.StringUtils; |
|
|
|
|
import com.fr.workspace.resource.ResourceIOException; |
|
|
|
|
|
|
|
|
|
import java.io.ByteArrayInputStream; |
|
|
|
@ -48,14 +48,10 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard() |
|
|
|
|
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.getEndPoint(), config.getRegion())) |
|
|
|
|
.withCredentials(new AWSStaticCredentialsProvider(credentials)).disableChunkedEncoding(); |
|
|
|
|
if (config.isEnablePathStyleAccess()) { |
|
|
|
|
amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess(); |
|
|
|
|
} |
|
|
|
|
ClientConfiguration clientConfiguration = new ClientConfiguration(); |
|
|
|
|
if (StringUtils.isNotEmpty(config.getSignerOverride())) { |
|
|
|
|
clientConfiguration.setSignerOverride(config.getSignerOverride()); |
|
|
|
|
} |
|
|
|
|
clientConfiguration.setProtocol(Protocol.HTTP); |
|
|
|
|
clientConfiguration.setUseGzip(config.getUseGzip()); |
|
|
|
|
clientConfiguration.setUseTcpKeepAlive(config.getKeepAlive()); |
|
|
|
|
clientConfiguration.setMaxConnections(config.getMaxConnection()); |
|
|
|
|
amazonS3ClientBuilder = amazonS3ClientBuilder.withClientConfiguration(clientConfiguration); |
|
|
|
|
this.s3 = amazonS3ClientBuilder.build(); |
|
|
|
|
this.bucket = config.getBucket(); |
|
|
|
@ -63,7 +59,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public String getSeparator() { |
|
|
|
|
return "/"; |
|
|
|
|
return DELIMITER; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
@ -72,10 +68,9 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
ObjectMetadata metadata = s3.getObjectMetadata(bucket, path); |
|
|
|
|
return s3Object2FileEntry(metadata, path); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.info("{} not exist!", path); |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "{} not exist!", path); |
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return null; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
@ -84,21 +79,26 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
if (!dir.endsWith(DELIMITER)) { |
|
|
|
|
dir += DELIMITER; |
|
|
|
|
} |
|
|
|
|
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) |
|
|
|
|
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(this.bucket) |
|
|
|
|
.withPrefix(dir).withDelimiter(DELIMITER); |
|
|
|
|
ObjectListing objectListing = s3.listObjects(listObjectsRequest); |
|
|
|
|
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { |
|
|
|
|
String key = summary.getKey(); |
|
|
|
|
if (!key.endsWith(DELIMITER)) { |
|
|
|
|
result.add(s3Object2FileEntry(summary, key)); |
|
|
|
|
String nextMarker = null; |
|
|
|
|
ObjectListing objectListing; |
|
|
|
|
do { |
|
|
|
|
objectListing = this.s3.listObjects(listObjectsRequest.withMarker(nextMarker)); |
|
|
|
|
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { |
|
|
|
|
String key = summary.getKey(); |
|
|
|
|
if (!key.endsWith(DELIMITER)) { |
|
|
|
|
result.add(s3Object2FileEntry(summary, key)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
for (String prefix : objectListing.getCommonPrefixes()) { |
|
|
|
|
FineFileEntry entry = new FineFileEntry(prefix); |
|
|
|
|
entry.setDirectory(true); |
|
|
|
|
result.add(entry); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
for (String prefix : objectListing.getCommonPrefixes()) { |
|
|
|
|
FineFileEntry entry = new FineFileEntry(prefix); |
|
|
|
|
entry.setDirectory(true); |
|
|
|
|
result.add(entry); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
nextMarker = objectListing.getNextMarker(); |
|
|
|
|
} while (objectListing.isTruncated()); |
|
|
|
|
return result.toArray(new FineFileEntry[0]); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -113,27 +113,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
try { |
|
|
|
|
return s3.getObject(request).getObjectContent(); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] read path {} is error!", filePath); |
|
|
|
|
return new ByteArrayInputStream(new byte[0]); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public void write(String path, byte[] data) { |
|
|
|
|
|
|
|
|
|
List<String> paths = getAllNecessaryPath(path); |
|
|
|
|
for (int i = paths.size() - 1; i > 0; i--) { |
|
|
|
|
String parent = paths.get(i); |
|
|
|
|
PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata()); |
|
|
|
|
try { |
|
|
|
|
s3.putObject(req); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.error("[S3] Failed to create parent path {}", parent); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
ObjectMetadata metadata; |
|
|
|
|
try { |
|
|
|
|
metadata = s3.getObjectMetadata(bucket, path); |
|
|
|
|
metadata = this.s3.getObjectMetadata(this.bucket, path); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
FineLoggerFactory.getLogger().warn("[S3] getObjectMetaData error ,path {}", path); |
|
|
|
|
metadata = new ObjectMetadata(); |
|
|
|
|
String mimeType = URLConnection.guessContentTypeFromName(path); |
|
|
|
|
if (mimeType != null) { |
|
|
|
@ -146,18 +137,38 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
s3.putObject(bucket, path, new ByteArrayInputStream(data), metadata); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public void write(String path, InputStream inputStream) throws ResourceIOException { |
|
|
|
|
if (null == inputStream) { |
|
|
|
|
this.s3.putObject(this.bucket, path, ""); |
|
|
|
|
} else { |
|
|
|
|
ObjectMetadata metadata = new ObjectMetadata(); |
|
|
|
|
try { |
|
|
|
|
metadata.setContentLength(inputStream.available()); |
|
|
|
|
this.s3.putObject(this.bucket, path, inputStream, metadata); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
FineLoggerFactory.getLogger().info("S3 inputStream.putObject error,path is {}, error message {}", path, e.getMessage()); |
|
|
|
|
} finally { |
|
|
|
|
try { |
|
|
|
|
inputStream.close(); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] error,path is {}, error message {}", path, e.getMessage()); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public boolean createFile(String path) { |
|
|
|
|
|
|
|
|
|
List<String> paths = getAllNecessaryPath(path); |
|
|
|
|
for (int i = paths.size() - 1; i >= 0; i--) { |
|
|
|
|
String parent = paths.get(i); |
|
|
|
|
PutObjectRequest req = new PutObjectRequest(bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata()); |
|
|
|
|
PutObjectRequest req = new PutObjectRequest(this.bucket, parent, new ByteArrayInputStream(new byte[0]), buildEmptyMetadata()); |
|
|
|
|
try { |
|
|
|
|
s3.putObject(req); |
|
|
|
|
this.s3.putObject(req); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.error("[S3] Failed to create parent path {}", parent); |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] Failed to create parent path {}", parent); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
@ -166,7 +177,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public boolean createDirectory(String path) { |
|
|
|
|
|
|
|
|
|
if (!path.endsWith(DELIMITER)) { |
|
|
|
|
path += DELIMITER; |
|
|
|
|
} |
|
|
|
@ -177,7 +187,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
try { |
|
|
|
|
s3.putObject(req); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.error("[S3] Failed to create parent path {}", parent); |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] Failed to create parent path {}", parent); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
@ -186,37 +196,42 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public boolean delete(String path) { |
|
|
|
|
|
|
|
|
|
s3.deleteObject(bucket, path); |
|
|
|
|
this.s3.deleteObject(this.bucket, path); |
|
|
|
|
String prefix = path; |
|
|
|
|
if (!path.endsWith(DELIMITER)) { |
|
|
|
|
prefix = path + DELIMITER; |
|
|
|
|
} |
|
|
|
|
try { |
|
|
|
|
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) |
|
|
|
|
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(this.bucket) |
|
|
|
|
.withPrefix(prefix).withDelimiter(DELIMITER); |
|
|
|
|
ObjectListing objectListing = s3.listObjects(listObjectsRequest); |
|
|
|
|
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { |
|
|
|
|
String key = summary.getKey(); |
|
|
|
|
if (!key.endsWith(DELIMITER)) { |
|
|
|
|
s3.deleteObject(bucket, key); |
|
|
|
|
ObjectListing objectListing; |
|
|
|
|
String nextMarker = null; |
|
|
|
|
do { |
|
|
|
|
objectListing = this.s3.listObjects(listObjectsRequest.withMarker(nextMarker)); |
|
|
|
|
for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { |
|
|
|
|
String key = summary.getKey(); |
|
|
|
|
if (!key.endsWith(DELIMITER)) { |
|
|
|
|
this.s3.deleteObject(this.bucket, key); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
for (String pre : objectListing.getCommonPrefixes()) { |
|
|
|
|
delete(pre); |
|
|
|
|
} |
|
|
|
|
for (String pre : objectListing.getCommonPrefixes()) { |
|
|
|
|
delete(pre); |
|
|
|
|
} |
|
|
|
|
nextMarker = objectListing.getNextMarker(); |
|
|
|
|
} while (objectListing.isTruncated()); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.error(e.getMessage(), e); |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] delete error path {}", path); |
|
|
|
|
} |
|
|
|
|
s3.deleteObject(bucket, prefix); |
|
|
|
|
this.s3.deleteObject(this.bucket, prefix); |
|
|
|
|
return true; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public boolean exist(String path) { |
|
|
|
|
try { |
|
|
|
|
return s3.doesObjectExist(bucket, path) || s3.doesObjectExist(bucket, path + DELIMITER); |
|
|
|
|
return this.s3.doesObjectExist(this.bucket, path) || this.s3.doesObjectExist(this.bucket, path + DELIMITER); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] exist error path {}", path); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
@ -227,36 +242,40 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
if (!dir.endsWith(DELIMITER)) { |
|
|
|
|
dir += DELIMITER; |
|
|
|
|
} |
|
|
|
|
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) |
|
|
|
|
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(this.bucket) |
|
|
|
|
.withPrefix(dir).withDelimiter(DELIMITER); |
|
|
|
|
ObjectListing objectListing = s3.listObjects(listObjectsRequest); |
|
|
|
|
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { |
|
|
|
|
|
|
|
|
|
String key = objectSummary.getKey(); |
|
|
|
|
if (StringKit.equals(key, dir)) { |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
String[] arr = key.split(DELIMITER); |
|
|
|
|
String name = arr[arr.length - 1]; |
|
|
|
|
if (filter == null) { |
|
|
|
|
result.add(name); |
|
|
|
|
} else { |
|
|
|
|
if (filter.accept(name)) { |
|
|
|
|
String nextMarker = null; |
|
|
|
|
ObjectListing objectListing; |
|
|
|
|
do { |
|
|
|
|
objectListing = this.s3.listObjects(listObjectsRequest.withMarker(nextMarker)); |
|
|
|
|
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { |
|
|
|
|
String key = objectSummary.getKey(); |
|
|
|
|
if (StringKit.equals(key, dir)) { |
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
|
String[] arr = key.split(DELIMITER); |
|
|
|
|
String name = arr[arr.length - 1]; |
|
|
|
|
if (filter == null) { |
|
|
|
|
result.add(name); |
|
|
|
|
} else { |
|
|
|
|
if (filter.accept(name)) { |
|
|
|
|
result.add(name); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
for (String prefix : objectListing.getCommonPrefixes()) { |
|
|
|
|
String[] arr = prefix.split(DELIMITER); |
|
|
|
|
String name = arr[arr.length - 1] + DELIMITER; |
|
|
|
|
if (filter == null) { |
|
|
|
|
result.add(name); |
|
|
|
|
} else { |
|
|
|
|
if (filter.accept(name)) { |
|
|
|
|
for (String prefix : objectListing.getCommonPrefixes()) { |
|
|
|
|
String[] arr = prefix.split(DELIMITER); |
|
|
|
|
String name = arr[arr.length - 1] + DELIMITER; |
|
|
|
|
if (filter == null) { |
|
|
|
|
result.add(name); |
|
|
|
|
} else { |
|
|
|
|
if (filter.accept(name)) { |
|
|
|
|
result.add(name); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
nextMarker = objectListing.getNextMarker(); |
|
|
|
|
} while (objectListing.isTruncated()); |
|
|
|
|
return result.toArray(new String[0]); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
@ -268,7 +287,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
@Override |
|
|
|
|
public long lastModified(String path) { |
|
|
|
|
try { |
|
|
|
|
S3Object s3Object = s3.getObject(bucket, path); |
|
|
|
|
S3Object s3Object = this.s3.getObject(this.bucket, path); |
|
|
|
|
if (s3Object != null) { |
|
|
|
|
try { |
|
|
|
|
return s3Object.getObjectMetadata().getLastModified().getTime(); |
|
|
|
@ -277,19 +296,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.info("{} not exist!", path); |
|
|
|
|
FineLoggerFactory.getLogger().error(e, "[S3] {} not exist!", path); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return -1L; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public long length(String path) { |
|
|
|
|
try { |
|
|
|
|
ObjectMetadata metadata = s3.getObjectMetadata(bucket, path); |
|
|
|
|
ObjectMetadata metadata = this.s3.getObjectMetadata(this.bucket, path); |
|
|
|
|
return metadata.getContentLength(); |
|
|
|
|
} catch (Exception e) { |
|
|
|
|
LogKit.info("{} not exist!", path); |
|
|
|
|
LogKit.info("[S3] {} not exist!", path); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return -1L; |
|
|
|
@ -297,15 +315,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
|
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
public boolean rename(String path, String newPath) throws ResourceIOException { |
|
|
|
|
|
|
|
|
|
if (copy(path, newPath) && delete(path)) { |
|
|
|
|
if (LogKit.isDebugEnabled()) { |
|
|
|
|
LogKit.debug("[S3] rename {} to {} success.", path, newPath); |
|
|
|
|
if (FineLoggerFactory.getLogger().isDebugEnabled()) { |
|
|
|
|
FineLoggerFactory.getLogger().debug("[S3] rename {} to {} success.", path, newPath); |
|
|
|
|
} |
|
|
|
|
return true; |
|
|
|
|
} else { |
|
|
|
|
FineLoggerFactory.getLogger().error("[S3] rename {} to {} failed.", path, newPath); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
LogKit.error("[S3] rename {} to {} failed.", path, newPath); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|