Browse Source

[Fix-10665] [S3] Fix s3 download method (#10675)

when overwrite param is true and dest file is exist, throw exception
make dirs for ancestor directories of the source file path if it is not exists
To describe the error more clearly, change "the file isn`t exists" to specific error message

Co-authored-by: sheldonliu <sheldonliu>
(cherry picked from commit efd68f4805)
3.0.0/version-upgrade
Sheldon 2 years ago committed by Jiajie Zhong
parent
commit
1f55cc89c1
  1. 18
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java

18
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java

@ -50,6 +50,7 @@ import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -162,21 +163,26 @@ public class S3Utils implements Closeable, StorageOperate {
} }
@Override @Override
public void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource, boolean overwrite) throws IOException { public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource, boolean overwrite) throws IOException {
File dstFile = new File(dstFilePath);
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
}
S3Object o = s3Client.getObject(BUCKET_NAME, srcFilePath); S3Object o = s3Client.getObject(BUCKET_NAME, srcFilePath);
try (S3ObjectInputStream s3is = o.getObjectContent(); try (S3ObjectInputStream s3is = o.getObjectContent();
FileOutputStream fos = new FileOutputStream(new File(dstFile))) { FileOutputStream fos = new FileOutputStream(dstFilePath)) {
byte[] readBuf = new byte[1024]; byte[] readBuf = new byte[1024];
int readLen = 0; int readLen;
while ((readLen = s3is.read(readBuf)) > 0) { while ((readLen = s3is.read(readBuf)) > 0) {
fos.write(readBuf, 0, readLen); fos.write(readBuf, 0, readLen);
} }
} catch (AmazonServiceException e) { } catch (AmazonServiceException e) {
logger.error("the resource can`t be downloaded,the bucket is {},and the src is {}", tenantCode, srcFilePath);
throw new IOException(e.getMessage()); throw new IOException(e.getMessage());
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
logger.error("the file isn`t exists"); logger.error("the destination file {} not found", dstFilePath);
throw new IOException("the file isn`t exists"); throw e;
} }
} }

Loading…
Cancel
Save