@ -18,8 +18,8 @@ import com.fanruan.api.log.LogKit;
import com.fanruan.api.util.StringKit ;
import com.fanruan.api.util.StringKit ;
import com.fr.io.repository.FineFileEntry ;
import com.fr.io.repository.FineFileEntry ;
import com.fr.io.repository.base.BaseResourceRepository ;
import com.fr.io.repository.base.BaseResourceRepository ;
import com.fr.log.FineLoggerFactory ;
import com.fr.stable.Filter ;
import com.fr.stable.Filter ;
import com.fr.stable.StringUtils ;
import com.fr.workspace.resource.ResourceIOException ;
import com.fr.workspace.resource.ResourceIOException ;
import java.io.ByteArrayInputStream ;
import java.io.ByteArrayInputStream ;
@ -48,14 +48,10 @@ public class S3ResourceRepository extends BaseResourceRepository {
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder . standard ( )
AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder . standard ( )
. withEndpointConfiguration ( new AwsClientBuilder . EndpointConfiguration ( config . getEndPoint ( ) , config . getRegion ( ) ) )
. withEndpointConfiguration ( new AwsClientBuilder . EndpointConfiguration ( config . getEndPoint ( ) , config . getRegion ( ) ) )
. withCredentials ( new AWSStaticCredentialsProvider ( credentials ) ) . disableChunkedEncoding ( ) ;
. withCredentials ( new AWSStaticCredentialsProvider ( credentials ) ) . disableChunkedEncoding ( ) ;
if ( config . isEnablePathStyleAccess ( ) ) {
amazonS3ClientBuilder = amazonS3ClientBuilder . enablePathStyleAccess ( ) ;
}
ClientConfiguration clientConfiguration = new ClientConfiguration ( ) ;
ClientConfiguration clientConfiguration = new ClientConfiguration ( ) ;
if ( StringUtils . isNotEmpty ( config . getSignerOverride ( ) ) ) {
clientConfiguration . setUseGzip ( config . getUseGzip ( ) ) ;
clientConfiguration . setSignerOverride ( config . getSignerOverride ( ) ) ;
clientConfiguration . setUseTcpKeepAlive ( config . getKeepAlive ( ) ) ;
}
clientConfiguration . setMaxConnections ( config . getMaxConnection ( ) ) ;
clientConfiguration . setProtocol ( Protocol . HTTP ) ;
amazonS3ClientBuilder = amazonS3ClientBuilder . withClientConfiguration ( clientConfiguration ) ;
amazonS3ClientBuilder = amazonS3ClientBuilder . withClientConfiguration ( clientConfiguration ) ;
this . s3 = amazonS3ClientBuilder . build ( ) ;
this . s3 = amazonS3ClientBuilder . build ( ) ;
this . bucket = config . getBucket ( ) ;
this . bucket = config . getBucket ( ) ;
@ -63,7 +59,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
@Override
public String getSeparator ( ) {
public String getSeparator ( ) {
return "/" ;
return DELIMITER ;
}
}
@Override
@Override
@ -72,10 +68,9 @@ public class S3ResourceRepository extends BaseResourceRepository {
ObjectMetadata metadata = s3 . getObjectMetadata ( bucket , path ) ;
ObjectMetadata metadata = s3 . getObjectMetadata ( bucket , path ) ;
return s3Object2FileEntry ( metadata , path ) ;
return s3Object2FileEntry ( metadata , path ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
LogKit . info ( "{} not exist!" , path ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "{} not exist!" , path ) ;
return null ;
}
}
return null ;
}
}
@Override
@Override
@ -84,21 +79,26 @@ public class S3ResourceRepository extends BaseResourceRepository {
if ( ! dir . endsWith ( DELIMITER ) ) {
if ( ! dir . endsWith ( DELIMITER ) ) {
dir + = DELIMITER ;
dir + = DELIMITER ;
}
}
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( bucket )
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( this . bucket )
. withPrefix ( dir ) . withDelimiter ( DELIMITER ) ;
. withPrefix ( dir ) . withDelimiter ( DELIMITER ) ;
ObjectListing objectListing = s3 . listObjects ( listObjectsRequest ) ;
String nextMarker = null ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
ObjectListing objectListing ;
String key = summary . getKey ( ) ;
do {
if ( ! key . endsWith ( DELIMITER ) ) {
objectListing = this . s3 . listObjects ( listObjectsRequest . withMarker ( nextMarker ) ) ;
result . add ( s3Object2FileEntry ( summary , key ) ) ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String key = summary . getKey ( ) ;
if ( ! key . endsWith ( DELIMITER ) ) {
result . add ( s3Object2FileEntry ( summary , key ) ) ;
}
}
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
FineFileEntry entry = new FineFileEntry ( prefix ) ;
entry . setDirectory ( true ) ;
result . add ( entry ) ;
}
}
}
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
FineFileEntry entry = new FineFileEntry ( prefix ) ;
entry . setDirectory ( true ) ;
result . add ( entry ) ;
}
nextMarker = objectListing . getNextMarker ( ) ;
} while ( objectListing . isTruncated ( ) ) ;
return result . toArray ( new FineFileEntry [ 0 ] ) ;
return result . toArray ( new FineFileEntry [ 0 ] ) ;
}
}
@ -113,27 +113,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
try {
return s3 . getObject ( request ) . getObjectContent ( ) ;
return s3 . getObject ( request ) . getObjectContent ( ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] read path {} is error!" , filePath ) ;
return new ByteArrayInputStream ( new byte [ 0 ] ) ;
return new ByteArrayInputStream ( new byte [ 0 ] ) ;
}
}
}
}
@Override
@Override
public void write ( String path , byte [ ] data ) {
public void write ( String path , byte [ ] data ) {
List < String > paths = getAllNecessaryPath ( path ) ;
for ( int i = paths . size ( ) - 1 ; i > 0 ; i - - ) {
String parent = paths . get ( i ) ;
PutObjectRequest req = new PutObjectRequest ( bucket , parent , new ByteArrayInputStream ( new byte [ 0 ] ) , buildEmptyMetadata ( ) ) ;
try {
s3 . putObject ( req ) ;
} catch ( Exception e ) {
LogKit . error ( "[S3] Failed to create parent path {}" , parent ) ;
}
}
ObjectMetadata metadata ;
ObjectMetadata metadata ;
try {
try {
metadata = s3 . getObjectMetadata ( bucket , path ) ;
metadata = this . s3 . getObjectMetadata ( this . bucket , path ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . warn ( "[S3] getObjectMetaData error ,path {}" , path ) ;
metadata = new ObjectMetadata ( ) ;
metadata = new ObjectMetadata ( ) ;
String mimeType = URLConnection . guessContentTypeFromName ( path ) ;
String mimeType = URLConnection . guessContentTypeFromName ( path ) ;
if ( mimeType ! = null ) {
if ( mimeType ! = null ) {
@ -146,18 +137,38 @@ public class S3ResourceRepository extends BaseResourceRepository {
s3 . putObject ( bucket , path , new ByteArrayInputStream ( data ) , metadata ) ;
s3 . putObject ( bucket , path , new ByteArrayInputStream ( data ) , metadata ) ;
}
}
@Override
public void write ( String path , InputStream inputStream ) throws ResourceIOException {
if ( null = = inputStream ) {
this . s3 . putObject ( this . bucket , path , "" ) ;
} else {
ObjectMetadata metadata = new ObjectMetadata ( ) ;
try {
metadata . setContentLength ( inputStream . available ( ) ) ;
this . s3 . putObject ( this . bucket , path , inputStream , metadata ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . info ( "S3 inputStream.putObject error,path is {}, error message {}" , path , e . getMessage ( ) ) ;
} finally {
try {
inputStream . close ( ) ;
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] error,path is {}, error message {}" , path , e . getMessage ( ) ) ;
}
}
}
}
@Override
@Override
public boolean createFile ( String path ) {
public boolean createFile ( String path ) {
List < String > paths = getAllNecessaryPath ( path ) ;
List < String > paths = getAllNecessaryPath ( path ) ;
for ( int i = paths . size ( ) - 1 ; i > = 0 ; i - - ) {
for ( int i = paths . size ( ) - 1 ; i > = 0 ; i - - ) {
String parent = paths . get ( i ) ;
String parent = paths . get ( i ) ;
PutObjectRequest req = new PutObjectRequest ( bucket , parent , new ByteArrayInputStream ( new byte [ 0 ] ) , buildEmptyMetadata ( ) ) ;
PutObjectRequest req = new PutObjectRequest ( this . bucket , parent , new ByteArrayInputStream ( new byte [ 0 ] ) , buildEmptyMetadata ( ) ) ;
try {
try {
s3 . putObject ( req ) ;
this . s3 . putObject ( req ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
LogKit . error ( "[S3] Failed to create parent path {}" , parent ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] Failed to create parent path {}" , parent ) ;
return false ;
return false ;
}
}
}
}
@ -166,7 +177,6 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
@Override
public boolean createDirectory ( String path ) {
public boolean createDirectory ( String path ) {
if ( ! path . endsWith ( DELIMITER ) ) {
if ( ! path . endsWith ( DELIMITER ) ) {
path + = DELIMITER ;
path + = DELIMITER ;
}
}
@ -177,7 +187,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
try {
try {
s3 . putObject ( req ) ;
s3 . putObject ( req ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
LogKit . error ( "[S3] Failed to create parent path {}" , parent ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] Failed to create parent path {}" , parent ) ;
return false ;
return false ;
}
}
}
}
@ -186,37 +196,42 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
@Override
public boolean delete ( String path ) {
public boolean delete ( String path ) {
this . s3 . deleteObject ( this . bucket , path ) ;
s3 . deleteObject ( bucket , path ) ;
String prefix = path ;
String prefix = path ;
if ( ! path . endsWith ( DELIMITER ) ) {
if ( ! path . endsWith ( DELIMITER ) ) {
prefix = path + DELIMITER ;
prefix = path + DELIMITER ;
}
}
try {
try {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( bucket )
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( this . bucket )
. withPrefix ( prefix ) . withDelimiter ( DELIMITER ) ;
. withPrefix ( prefix ) . withDelimiter ( DELIMITER ) ;
ObjectListing objectListing = s3 . listObjects ( listObjectsRequest ) ;
ObjectListing objectListing ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String nextMarker = null ;
String key = summary . getKey ( ) ;
do {
if ( ! key . endsWith ( DELIMITER ) ) {
objectListing = this . s3 . listObjects ( listObjectsRequest . withMarker ( nextMarker ) ) ;
s3 . deleteObject ( bucket , key ) ;
for ( S3ObjectSummary summary : objectListing . getObjectSummaries ( ) ) {
String key = summary . getKey ( ) ;
if ( ! key . endsWith ( DELIMITER ) ) {
this . s3 . deleteObject ( this . bucket , key ) ;
}
}
}
}
for ( String pre : objectListing . getCommonPrefixes ( ) ) {
for ( String pre : objectListing . getCommonPrefixes ( ) ) {
delete ( pre ) ;
delete ( pre ) ;
}
}
nextMarker = objectListing . getNextMarker ( ) ;
} while ( objectListing . isTruncated ( ) ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
LogKit . error ( e . getMessage ( ) , e ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] delete error path {}" , path ) ;
}
}
s3 . deleteObject ( bucket , prefix ) ;
this . s3 . deleteObject ( this . bucket , prefix ) ;
return true ;
return true ;
}
}
@Override
@Override
public boolean exist ( String path ) {
public boolean exist ( String path ) {
try {
try {
return s3 . doesObjectExist ( bucket , path ) | | s3 . doesObjectExist ( bucket , path + DELIMITER ) ;
return this . s3 . doesObjectExist ( this . bucket , path ) | | this . s3 . doesObjectExist ( this . bucket , path + DELIMITER ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] exist error path {}" , path ) ;
return false ;
return false ;
}
}
}
}
@ -227,36 +242,40 @@ public class S3ResourceRepository extends BaseResourceRepository {
if ( ! dir . endsWith ( DELIMITER ) ) {
if ( ! dir . endsWith ( DELIMITER ) ) {
dir + = DELIMITER ;
dir + = DELIMITER ;
}
}
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( bucket )
ListObjectsRequest listObjectsRequest = new ListObjectsRequest ( ) . withBucketName ( this . bucket )
. withPrefix ( dir ) . withDelimiter ( DELIMITER ) ;
. withPrefix ( dir ) . withDelimiter ( DELIMITER ) ;
ObjectListing objectListing = s3 . listObjects ( listObjectsRequest ) ;
String nextMarker = null ;
for ( S3ObjectSummary objectSummary : objectListing . getObjectSummaries ( ) ) {
ObjectListing objectListing ;
do {
String key = objectSummary . getKey ( ) ;
objectListing = this . s3 . listObjects ( listObjectsRequest . withMarker ( nextMarker ) ) ;
if ( StringKit . equals ( key , dir ) ) {
for ( S3ObjectSummary objectSummary : objectListing . getObjectSummaries ( ) ) {
continue ;
String key = objectSummary . getKey ( ) ;
}
if ( StringKit . equals ( key , dir ) ) {
String [ ] arr = key . split ( DELIMITER ) ;
continue ;
String name = arr [ arr . length - 1 ] ;
}
if ( filter = = null ) {
String [ ] arr = key . split ( DELIMITER ) ;
result . add ( name ) ;
String name = arr [ arr . length - 1 ] ;
} else {
if ( filter = = null ) {
if ( filter . accept ( name ) ) {
result . add ( name ) ;
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
result . add ( name ) ;
}
}
}
}
}
}
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
for ( String prefix : objectListing . getCommonPrefixes ( ) ) {
String [ ] arr = prefix . split ( DELIMITER ) ;
String [ ] arr = prefix . split ( DELIMITER ) ;
String name = arr [ arr . length - 1 ] + DELIMITER ;
String name = arr [ arr . length - 1 ] + DELIMITER ;
if ( filter = = null ) {
if ( filter = = null ) {
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
result . add ( name ) ;
result . add ( name ) ;
} else {
if ( filter . accept ( name ) ) {
result . add ( name ) ;
}
}
}
}
}
}
nextMarker = objectListing . getNextMarker ( ) ;
} while ( objectListing . isTruncated ( ) ) ;
return result . toArray ( new String [ 0 ] ) ;
return result . toArray ( new String [ 0 ] ) ;
}
}
@ -268,7 +287,7 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
@Override
public long lastModified ( String path ) {
public long lastModified ( String path ) {
try {
try {
S3Object s3Object = s3 . getObject ( bucket , path ) ;
S3Object s3Object = this . s3 . getObject ( this . bucket , path ) ;
if ( s3Object ! = null ) {
if ( s3Object ! = null ) {
try {
try {
return s3Object . getObjectMetadata ( ) . getLastModified ( ) . getTime ( ) ;
return s3Object . getObjectMetadata ( ) . getLastModified ( ) . getTime ( ) ;
@ -277,19 +296,18 @@ public class S3ResourceRepository extends BaseResourceRepository {
}
}
}
}
} catch ( Exception e ) {
} catch ( Exception e ) {
LogKit . info ( " {} not exist!", path ) ;
FineLoggerFactory . getLogger ( ) . error ( e , "[S3] {} not exist!", path ) ;
}
}
return - 1L ;
return - 1L ;
}
}
@Override
@Override
public long length ( String path ) {
public long length ( String path ) {
try {
try {
ObjectMetadata metadata = s3 . getObjectMetadata ( bucket , path ) ;
ObjectMetadata metadata = this . s3 . getObjectMetadata ( this . bucket , path ) ;
return metadata . getContentLength ( ) ;
return metadata . getContentLength ( ) ;
} catch ( Exception e ) {
} catch ( Exception e ) {
LogKit . info ( "{} not exist!" , path ) ;
LogKit . info ( "[S3] {} not exist!" , path ) ;
}
}
return - 1L ;
return - 1L ;
@ -297,15 +315,15 @@ public class S3ResourceRepository extends BaseResourceRepository {
@Override
@Override
public boolean rename ( String path , String newPath ) throws ResourceIOException {
public boolean rename ( String path , String newPath ) throws ResourceIOException {
if ( copy ( path , newPath ) & & delete ( path ) ) {
if ( copy ( path , newPath ) & & delete ( path ) ) {
if ( LogKit . isDebugEnabled ( ) ) {
if ( FineLoggerFactory . getLogger ( ) . isDebugEnabled ( ) ) {
LogKit . debug ( "[S3] rename {} to {} success." , path , newPath ) ;
FineLoggerFactory . getLogger ( ) . debug ( "[S3] rename {} to {} success." , path , newPath ) ;
}
}
return true ;
return true ;
} else {
FineLoggerFactory . getLogger ( ) . error ( "[S3] rename {} to {} failed." , path , newPath ) ;
return false ;
}
}
LogKit . error ( "[S3] rename {} to {} failed." , path , newPath ) ;
return false ;
}
}
@Override
@Override