@ -18,8 +18,8 @@ import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit ;
import com.fr.io.repository.FineFileEntry ;
import com.fr.io.repository.base.BaseResourceRepository ;
import com.fr.log.FineLoggerFactory ;
import com.fr.stable.Filter ;
import com.fr.stable.StringUtils ;
import com.fr.workspace.resource.ResourceIOException ;
import java.io.ByteArrayInputStream ;
@ -48,14 +48,10 @@ public class S3ResourceRepository extends BaseResourceRepository {
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder . standard ( )
. withEndpointConfiguration ( new AwsClientBuilder . EndpointConfiguration ( config . getEndPoint ( ) , config . getRegion ( ) ) )
. withCredentials ( new AWSStaticCredentialsProvider ( credentials ) ) . disableChunkedEncoding ( ) ;
if ( config . isEnablePathStyleAccess ( ) ) {
amazonS3ClientBuilder = amazonS3ClientBuilder . enablePathStyleAccess ( ) ;
}
ClientConfiguration clientConfiguration = new ClientConfiguration ( ) ;
if ( StringUtils . isNotEmpty ( config . getSignerOverride ( ) ) ) {
clientConfiguration . setSignerOverride ( config . getSignerOverride ( ) ) ;
}
clientConfiguration . setProtocol ( Protocol . HTTP ) ;
clientConfiguration . setUseGzip ( config . getUseGzip ( ) ) ;
clientConfiguration . setUseTcpKeepAlive ( config . getKeepAlive ( ) ) ;
clientConfiguration . setMaxConnections ( config . getMaxConnection ( ) ) ;
amazonS3ClientBuilder = amazonS3ClientBuilder . withClientConfiguration ( clientConfiguration ) ;
this . s3 = amazonS3ClientBuilder . build ( ) ;
this . bucket = config . getBucket ( ) ;
@ -63,7 +59,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public String getSeparator ( ) {
return "/" ;
return DELIMITER ;
}
@Override
@ -72,10 +68,9 @@ public class S3ResourceRepository extends BaseResourceRepository {
ObjectMetadata metadata = s3 . getObjectMetadata ( bucket , path ) ;
return s3Object2FileEntry ( metadata , path ) ;
} catch ( Exception e ) {
LogKit . info ( "{} not exist!" , path ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "{} not exist!" , path ) ;
return null ;
}
return null ;
}
@Override
@ -84,21 +79,26 @@ public class S3ResourceRepository extends BaseResourceRepository {
if ( ! dir . endsWith ( DELIMITER ) ) {
dir + = DELIMITER ;
}
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( bucket )
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( this . bucket )
. withPrefix ( dir ) . withDelimiter ( DELIMITER ) ;
ObjectListing objectListing = s3 . listObjects ( listObjectsRequest ) ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String key = summary . getKey ( ) ;
if ( ! key . endsWith ( DELIMITER ) ) {
result . add ( s3Object2FileEntry ( summary , key ) ) ;
String nextMarker = null ;
ObjectListing objectListing ;
do {
objectListing = this . s3 . listObjects ( listObjectsRequest . withMarker ( nextMarker ) ) ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String key = summary . getKey ( ) ;
if ( ! key . endsWith ( DELIMITER ) ) {
result . add ( s3Object2FileEntry ( summary , key ) ) ;
}
}
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
FineFileEntry entry = new FineFileEntry ( prefix ) ;
entry . setDirectory ( true ) ;
result . add ( entry ) ;
}
}
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
FineFileEntry entry = new FineFileEntry ( prefix ) ;
entry . setDirectory ( true ) ;
result . add ( entry ) ;
}
nextMarker = objectListing . getNextMarker ( ) ;
} while ( objectListing . isTruncated ( ) ) ;
return result . toArray ( new FineFileEntry [ 0 ] ) ;
}
@ -113,27 +113,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
return s3 . getObject ( request ) . getObjectContent ( ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] read path {} is error!" , filePath ) ;
return new ByteArrayInputStream ( new byte [ 0 ] ) ;
}
}
@Override
public void write ( String path , byte [ ] data ) {
List < String > paths = getAllNecessaryPath ( path ) ;
for ( int i = paths . size ( ) - 1 ; i > 0 ; i - - ) {
String parent = paths . get ( i ) ;
PutObjectRequest req = new PutObjectRequest ( bucket , parent , new ByteArrayInputStream ( new byte [ 0 ] ) , buildEmptyMetadata ( ) ) ;
try {
s3 . putObject ( req ) ;
} catch ( Exception e ) {
LogKit . error ( "[S3] Failed to create parent path {}" , parent ) ;
}
}
ObjectMetadata metadata ;
try {
metadata = s3 . getObjectMetadata ( bucket , path ) ;
metadata = this . s3 . getObjectMetadata ( this . bucket , path ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . warn ( "[S3] getObjectMetaData error ,path {}" , path ) ;
metadata = new ObjectMetadata ( ) ;
String mimeType = URLConnection . guessContentTypeFromName ( path ) ;
if ( mimeType ! = null ) {
@ -146,18 +137,38 @@ public class S3ResourceRepository extends BaseResourceRepository {
s3 . putObject ( bucket , path , new ByteArrayInputStream ( data ) , metadata ) ;
}
@Override
public void write ( String path , InputStream inputStream ) throws ResourceIOException {
if ( null = = inputStream ) {
this . s3 . putObject ( this . bucket , path , "" ) ;
} else {
ObjectMetadata metadata = new ObjectMetadata ( ) ;
try {
metadata . setContentLength ( inputStream . available ( ) ) ;
this . s3 . putObject ( this . bucket , path , inputStream , metadata ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . info ( "S3 inputStream.putObject error,path is {}, error message {}" , path , e . getMessage ( ) ) ;
} finally {
try {
inputStream . close ( ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] error,path is {}, error message {}" , path , e . getMessage ( ) ) ;
}
}
}
}
@Override
public boolean createFile ( String path ) {
List < String > paths = getAllNecessaryPath ( path ) ;
for ( int i = paths . size ( ) - 1 ; i > = 0 ; i - - ) {
String parent = paths . get ( i ) ;
PutObjectRequest req = new PutObjectRequest ( bucket , parent , new ByteArrayInputStream ( new byte [ 0 ] ) , buildEmptyMetadata ( ) ) ;
PutObjectRequest req = new PutObjectRequest ( this . bucket , parent , new ByteArrayInputStream ( new byte [ 0 ] ) , buildEmptyMetadata ( ) ) ;
try {
s3 . putObject ( req ) ;
this . s3 . putObject ( req ) ;
} catch ( Exception e ) {
LogKit . error ( "[S3] Failed to create parent path {}" , parent ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] Failed to create parent path {}" , parent ) ;
return false ;
}
}
@ -166,7 +177,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean createDirectory ( String path ) {
if ( ! path . endsWith ( DELIMITER ) ) {
path + = DELIMITER ;
}
@ -177,7 +187,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
s3 . putObject ( req ) ;
} catch ( Exception e ) {
LogKit . error ( "[S3] Failed to create parent path {}" , parent ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] Failed to create parent path {}" , parent ) ;
return false ;
}
}
@ -186,37 +196,42 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean delete ( String path ) {
s3 . deleteObject ( bucket , path ) ;
this . s3 . deleteObject ( this . bucket , path ) ;
String prefix = path ;
if ( ! path . endsWith ( DELIMITER ) ) {
prefix = path + DELIMITER ;
}
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( bucket )
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( this . bucket )
. withPrefix ( prefix ) . withDelimiter ( DELIMITER ) ;
ObjectListing objectListing = s3 . listObjects ( listObjectsRequest ) ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String key = summary . getKey ( ) ;
if ( ! key . endsWith ( DELIMITER ) ) {
s3 . deleteObject ( bucket , key ) ;
ObjectListing objectListing ;
String nextMarker = null ;
do {
objectListing = this . s3 . listObjects ( listObjectsRequest . withMarker ( nextMarker ) ) ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String key = summary . getKey ( ) ;
if ( ! key . endsWith ( DELIMITER ) ) {
this . s3 . deleteObject ( this . bucket , key ) ;
}
}
}
for ( String pre : objectListing . getCommonPrefixes ( ) ) {
delete ( pre ) ;
}
for ( String pre : objectListing . getCommonPrefixes ( ) ) {
delete ( pre ) ;
}
nextMarker = objectListing . getNextMarker ( ) ;
} while ( objectListing . isTruncated ( ) ) ;
} catch ( Exception e ) {
LogKit . error ( e . getMessage ( ) , e ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] delete error path {}" , path ) ;
}
s3 . deleteObject ( bucket , prefix ) ;
this . s3 . deleteObject ( this . bucket , prefix ) ;
return true ;
}
@Override
public boolean exist ( String path ) {
try {
return s3 . doesObjectExist ( bucket , path ) | | s3 . doesObjectExist ( bucket , path + DELIMITER ) ;
return this . s3 . doesObjectExist ( this . bucket , path ) | | this . s3 . doesObjectExist ( this . bucket , path + DELIMITER ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] exist error path {}" , path ) ;
return false ;
}
}
@ -227,36 +242,40 @@ public class S3ResourceRepository extends BaseResourceRepository {
if ( ! dir . endsWith ( DELIMITER ) ) {
dir + = DELIMITER ;
}
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( bucket )
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( this . bucket )
. withPrefix ( dir ) . withDelimiter ( DELIMITER ) ;
ObjectListing objectListing = s3 . listObjects ( listObjectsRequest ) ;
for ( S3ObjectSummary objectSummary : objectListing . getObjectSummaries ( ) ) {
String key = objectSummary . getKey ( ) ;
if ( StringKit . equals ( key , dir ) ) {
continue ;
}
String [ ] arr = key . split ( DELIMITER ) ;
String name = arr [ arr . length - 1 ] ;
if ( filter = = null ) {
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
String nextMarker = null ;
ObjectListing objectListing ;
do {
objectListing = this . s3 . listObjects ( listObjectsRequest . withMarker ( nextMarker ) ) ;
for ( S3ObjectSummary objectSummary : objectListing . getObjectSummaries ( ) ) {
String key = objectSummary . getKey ( ) ;
if ( StringKit . equals ( key , dir ) ) {
continue ;
}
String [ ] arr = key . split ( DELIMITER ) ;
String name = arr [ arr . length - 1 ] ;
if ( filter = = null ) {
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
result . add ( name ) ;
}
}
}
}
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
String [ ] arr = prefix . split ( DELIMITER ) ;
String name = arr [ arr . length - 1 ] + DELIMITER ;
if ( filter = = null ) {
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
String [ ] arr = prefix . split ( DELIMITER ) ;
String name = arr [ arr . length - 1 ] + DELIMITER ;
if ( filter = = null ) {
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
result . add ( name ) ;
}
}
}
}
nextMarker = objectListing . getNextMarker ( ) ;
} while ( objectListing . isTruncated ( ) ) ;
return result . toArray ( new String [ 0 ] ) ;
}
@ -268,7 +287,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public long lastModified ( String path ) {
try {
S3Object s3Object = s3 . getObject ( bucket , path ) ;
S3Object s3Object = this . s3 . getObject ( this . bucket , path ) ;
if ( s3Object ! = null ) {
try {
return s3Object . getObjectMetadata ( ) . getLastModified ( ) . getTime ( ) ;
@ -277,19 +296,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
}
}
} catch ( Exception e ) {
LogKit . info ( " {} not exist!", path ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] {} not exist!", path ) ;
}
return - 1L ;
}
@Override
public long length ( String path ) {
try {
ObjectMetadata metadata = s3 . getObjectMetadata ( bucket , path ) ;
ObjectMetadata metadata = this . s3 . getObjectMetadata ( this . bucket , path ) ;
return metadata . getContentLength ( ) ;
} catch ( Exception e ) {
LogKit . info ( "{} not exist!" , path ) ;
LogKit . info ( "[S3] {} not exist!" , path ) ;
}
return - 1L ;
@ -297,15 +315,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
public boolean rename ( String path , String newPath ) throws ResourceIOException {
if ( copy ( path , newPath ) & & delete ( path ) ) {
if ( LogKit . isDebugEnabled ( ) ) {
LogKit . debug ( "[S3] rename {} to {} success." , path , newPath ) ;
if ( FineLoggerFactory . getLogger ( ) . isDebugEnabled ( ) ) {
FineLoggerFactory . getLogger ( ) . debug ( "[S3] rename {} to {} success." , path , newPath ) ;
}
return true ;
} else {
FineLoggerFactory . getLogger ( ) . error ( "[S3] rename {} to {} failed." , path , newPath ) ;
return false ;
}
LogKit . error ( "[S3] rename {} to {} failed." , path , newPath ) ;
return false ;
}
@Override