Browse Source

[Fix-10665] [S3] Fix s3 download method (#10675)

when overwrite param is true and dest file is exist, throw exception
make dirs for ancestor directories of the source file path if it is not exists
To describe the error more clearly, change "the file isn`t exists" to specific error message

Co-authored-by: sheldonliu <sheldonliu>
3.1.0-release
Sheldon 2 years ago committed by GitHub
parent
commit
efd68f4805
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 18
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java

18
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java

@ -42,6 +42,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -175,21 +176,26 @@ public class S3Utils implements Closeable, StorageOperate {
}
@Override
public void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource, boolean overwrite) throws IOException {
public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource, boolean overwrite) throws IOException {
File dstFile = new File(dstFilePath);
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
}
S3Object o = s3Client.getObject(BUCKET_NAME, srcFilePath);
try (S3ObjectInputStream s3is = o.getObjectContent();
FileOutputStream fos = new FileOutputStream(dstFile)) {
FileOutputStream fos = new FileOutputStream(dstFilePath)) {
byte[] readBuf = new byte[1024];
int readLen = 0;
int readLen;
while ((readLen = s3is.read(readBuf)) > 0) {
fos.write(readBuf, 0, readLen);
}
} catch (AmazonServiceException e) {
logger.error("the resource can`t be downloaded,the bucket is {},and the src is {}", tenantCode, srcFilePath);
throw new IOException(e.getMessage());
} catch (FileNotFoundException e) {
logger.error("the file isn`t exists");
throw new IOException("the file isn`t exists");
logger.error("the destination file {} not found", dstFilePath);
throw e;
}
}

Loading…
Cancel
Save