Browse Source
* [Feature][Resource Center] Add support for Huawei Cloud OBS as storage of resource center * add license and add doc * add 3-party dependency license * Update LICENSE * fix * Update pom.xml * fix * fix --------- Co-authored-by: sunkang <sunkang@hjfruit.com> Co-authored-by: xiangzihao <460888207@qq.com> Co-authored-by: Rick Cheng <rickchengx@gmail.com>augit-log
Edison Catto
9 months ago
committed by
GitHub
18 changed files with 1173 additions and 7 deletions
@ -0,0 +1,202 @@
|
||||
|
||||
Apache License |
||||
Version 2.0, January 2004 |
||||
http://www.apache.org/licenses/ |
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||
|
||||
1. Definitions. |
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, |
||||
and distribution as defined by Sections 1 through 9 of this document. |
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by |
||||
the copyright owner that is granting the License. |
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all |
||||
other entities that control, are controlled by, or are under common |
||||
control with that entity. For the purposes of this definition, |
||||
"control" means (i) the power, direct or indirect, to cause the |
||||
direction or management of such entity, whether by contract or |
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||
outstanding shares, or (iii) beneficial ownership of such entity. |
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity |
||||
exercising permissions granted by this License. |
||||
|
||||
"Source" form shall mean the preferred form for making modifications, |
||||
including but not limited to software source code, documentation |
||||
source, and configuration files. |
||||
|
||||
"Object" form shall mean any form resulting from mechanical |
||||
transformation or translation of a Source form, including but |
||||
not limited to compiled object code, generated documentation, |
||||
and conversions to other media types. |
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or |
||||
Object form, made available under the License, as indicated by a |
||||
copyright notice that is included in or attached to the work |
||||
(an example is provided in the Appendix below). |
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object |
||||
form, that is based on (or derived from) the Work and for which the |
||||
editorial revisions, annotations, elaborations, or other modifications |
||||
represent, as a whole, an original work of authorship. For the purposes |
||||
of this License, Derivative Works shall not include works that remain |
||||
separable from, or merely link (or bind by name) to the interfaces of, |
||||
the Work and Derivative Works thereof. |
||||
|
||||
"Contribution" shall mean any work of authorship, including |
||||
the original version of the Work and any modifications or additions |
||||
to that Work or Derivative Works thereof, that is intentionally |
||||
submitted to Licensor for inclusion in the Work by the copyright owner |
||||
or by an individual or Legal Entity authorized to submit on behalf of |
||||
the copyright owner. For the purposes of this definition, "submitted" |
||||
means any form of electronic, verbal, or written communication sent |
||||
to the Licensor or its representatives, including but not limited to |
||||
communication on electronic mailing lists, source code control systems, |
||||
and issue tracking systems that are managed by, or on behalf of, the |
||||
Licensor for the purpose of discussing and improving the Work, but |
||||
excluding communication that is conspicuously marked or otherwise |
||||
designated in writing by the copyright owner as "Not a Contribution." |
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||
on behalf of whom a Contribution has been received by Licensor and |
||||
subsequently incorporated within the Work. |
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
copyright license to reproduce, prepare Derivative Works of, |
||||
publicly display, publicly perform, sublicense, and distribute the |
||||
Work and such Derivative Works in Source or Object form. |
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
(except as stated in this section) patent license to make, have made, |
||||
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||
where such license applies only to those patent claims licensable |
||||
by such Contributor that are necessarily infringed by their |
||||
Contribution(s) alone or by combination of their Contribution(s) |
||||
with the Work to which such Contribution(s) was submitted. If You |
||||
institute patent litigation against any entity (including a |
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||
or a Contribution incorporated within the Work constitutes direct |
||||
or contributory patent infringement, then any patent licenses |
||||
granted to You under this License for that Work shall terminate |
||||
as of the date such litigation is filed. |
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the |
||||
Work or Derivative Works thereof in any medium, with or without |
||||
modifications, and in Source or Object form, provided that You |
||||
meet the following conditions: |
||||
|
||||
(a) You must give any other recipients of the Work or |
||||
Derivative Works a copy of this License; and |
||||
|
||||
(b) You must cause any modified files to carry prominent notices |
||||
stating that You changed the files; and |
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works |
||||
that You distribute, all copyright, patent, trademark, and |
||||
attribution notices from the Source form of the Work, |
||||
excluding those notices that do not pertain to any part of |
||||
the Derivative Works; and |
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its |
||||
distribution, then any Derivative Works that You distribute must |
||||
include a readable copy of the attribution notices contained |
||||
within such NOTICE file, excluding those notices that do not |
||||
pertain to any part of the Derivative Works, in at least one |
||||
of the following places: within a NOTICE text file distributed |
||||
as part of the Derivative Works; within the Source form or |
||||
documentation, if provided along with the Derivative Works; or, |
||||
within a display generated by the Derivative Works, if and |
||||
wherever such third-party notices normally appear. The contents |
||||
of the NOTICE file are for informational purposes only and |
||||
do not modify the License. You may add Your own attribution |
||||
notices within Derivative Works that You distribute, alongside |
||||
or as an addendum to the NOTICE text from the Work, provided |
||||
that such additional attribution notices cannot be construed |
||||
as modifying the License. |
||||
|
||||
You may add Your own copyright statement to Your modifications and |
||||
may provide additional or different license terms and conditions |
||||
for use, reproduction, or distribution of Your modifications, or |
||||
for any such Derivative Works as a whole, provided Your use, |
||||
reproduction, and distribution of the Work otherwise complies with |
||||
the conditions stated in this License. |
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||
any Contribution intentionally submitted for inclusion in the Work |
||||
by You to the Licensor shall be under the terms and conditions of |
||||
this License, without any additional terms or conditions. |
||||
Notwithstanding the above, nothing herein shall supersede or modify |
||||
the terms of any separate license agreement you may have executed |
||||
with Licensor regarding such Contributions. |
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade |
||||
names, trademarks, service marks, or product names of the Licensor, |
||||
except as required for reasonable and customary use in describing the |
||||
origin of the Work and reproducing the content of the NOTICE file. |
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or |
||||
agreed to in writing, Licensor provides the Work (and each |
||||
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||
implied, including, without limitation, any warranties or conditions |
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||
appropriateness of using or redistributing the Work and assume any |
||||
risks associated with Your exercise of permissions under this License. |
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory, |
||||
whether in tort (including negligence), contract, or otherwise, |
||||
unless required by applicable law (such as deliberate and grossly |
||||
negligent acts) or agreed to in writing, shall any Contributor be |
||||
liable to You for damages, including any direct, indirect, special, |
||||
incidental, or consequential damages of any character arising as a |
||||
result of this License or out of the use or inability to use the |
||||
Work (including but not limited to damages for loss of goodwill, |
||||
work stoppage, computer failure or malfunction, or any and all |
||||
other commercial damages or losses), even if such Contributor |
||||
has been advised of the possibility of such damages. |
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing |
||||
the Work or Derivative Works thereof, You may choose to offer, |
||||
and charge a fee for, acceptance of support, warranty, indemnity, |
||||
or other liability obligations and/or rights consistent with this |
||||
License. However, in accepting such obligations, You may act only |
||||
on Your own behalf and on Your sole responsibility, not on behalf |
||||
of any other Contributor, and only if You agree to indemnify, |
||||
defend, and hold each Contributor harmless for any liability |
||||
incurred by, or claims asserted against, such Contributor by reason |
||||
of your accepting any such warranty or additional liability. |
||||
|
||||
END OF TERMS AND CONDITIONS |
||||
|
||||
APPENDIX: How to apply the Apache License to your work. |
||||
|
||||
To apply the Apache License to your work, attach the following |
||||
boilerplate notice, with the fields enclosed by brackets "[]" |
||||
replaced with your own identifying information. (Don't include |
||||
the brackets!) The text should be enclosed in the appropriate |
||||
comment syntax for the file format. We also recommend that a |
||||
file or class name and description of purpose be included on the |
||||
same "printed page" as the copyright notice for easier |
||||
identification within third-party archives. |
||||
|
||||
Copyright [yyyy] [name of copyright owner] |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
@ -0,0 +1,46 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
<modelVersion>4.0.0</modelVersion> |
||||
<parent> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-storage-plugin</artifactId> |
||||
<version>dev-SNAPSHOT</version> |
||||
</parent> |
||||
|
||||
<artifactId>dolphinscheduler-storage-obs</artifactId> |
||||
|
||||
<dependencies> |
||||
<dependency> |
||||
<groupId>com.huaweicloud</groupId> |
||||
<artifactId>esdk-obs-java-bundle</artifactId> |
||||
</dependency> |
||||
<dependency> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-storage-api</artifactId> |
||||
<version>${project.version}</version> |
||||
</dependency> |
||||
|
||||
<dependency> |
||||
<groupId>org.apache.dolphinscheduler</groupId> |
||||
<artifactId>dolphinscheduler-task-api</artifactId> |
||||
<version>${project.version}</version> |
||||
</dependency> |
||||
</dependencies> |
||||
</project> |
||||
@ -0,0 +1,513 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.storage.obs; |
||||
|
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; |
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; |
||||
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; |
||||
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; |
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants; |
||||
import org.apache.dolphinscheduler.common.enums.ResUploadType; |
||||
import org.apache.dolphinscheduler.common.utils.PropertyUtils; |
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; |
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; |
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; |
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType; |
||||
|
||||
import org.apache.commons.lang3.StringUtils; |
||||
|
||||
import java.io.BufferedReader; |
||||
import java.io.ByteArrayInputStream; |
||||
import java.io.Closeable; |
||||
import java.io.File; |
||||
import java.io.FileNotFoundException; |
||||
import java.io.FileOutputStream; |
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.InputStreamReader; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.Paths; |
||||
import java.util.ArrayList; |
||||
import java.util.Collections; |
||||
import java.util.LinkedList; |
||||
import java.util.List; |
||||
import java.util.stream.Collectors; |
||||
import java.util.stream.Stream; |
||||
|
||||
import lombok.Data; |
||||
import lombok.extern.slf4j.Slf4j; |
||||
|
||||
import com.obs.services.ObsClient; |
||||
import com.obs.services.exception.ObsException; |
||||
import com.obs.services.internal.ServiceException; |
||||
import com.obs.services.model.DeleteObjectsRequest; |
||||
import com.obs.services.model.GetObjectRequest; |
||||
import com.obs.services.model.ListObjectsRequest; |
||||
import com.obs.services.model.ObjectListing; |
||||
import com.obs.services.model.ObjectMetadata; |
||||
import com.obs.services.model.ObsObject; |
||||
import com.obs.services.model.PutObjectRequest; |
||||
|
||||
@Data |
||||
@Slf4j |
||||
public class ObsStorageOperator implements Closeable, StorageOperate { |
||||
|
||||
private String accessKeyId; |
||||
|
||||
private String accessKeySecret; |
||||
|
||||
private String bucketName; |
||||
|
||||
private String endPoint; |
||||
|
||||
private ObsClient obsClient; |
||||
|
||||
public ObsStorageOperator() { |
||||
} |
||||
|
||||
public void init() { |
||||
this.accessKeyId = readObsAccessKeyID(); |
||||
this.accessKeySecret = readObsAccessKeySecret(); |
||||
this.endPoint = readObsEndPoint(); |
||||
this.bucketName = readObsBucketName(); |
||||
this.obsClient = buildObsClient(); |
||||
ensureBucketSuccessfullyCreated(bucketName); |
||||
} |
||||
|
||||
protected String readObsAccessKeyID() { |
||||
return PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_ID); |
||||
} |
||||
|
||||
protected String readObsAccessKeySecret() { |
||||
return PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_SECRET); |
||||
} |
||||
|
||||
protected String readObsBucketName() { |
||||
return PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_BUCKET_NAME); |
||||
} |
||||
|
||||
protected String readObsEndPoint() { |
||||
return PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_END_POINT); |
||||
} |
||||
|
||||
@Override |
||||
public void close() throws IOException { |
||||
obsClient.close(); |
||||
} |
||||
|
||||
@Override |
||||
public void createTenantDirIfNotExists(String tenantCode) throws Exception { |
||||
mkdir(tenantCode, getObsResDir(tenantCode)); |
||||
mkdir(tenantCode, getObsUdfDir(tenantCode)); |
||||
} |
||||
|
||||
@Override |
||||
public String getResDir(String tenantCode) { |
||||
return getObsResDir(tenantCode) + FOLDER_SEPARATOR; |
||||
} |
||||
|
||||
@Override |
||||
public String getUdfDir(String tenantCode) { |
||||
return getObsUdfDir(tenantCode) + FOLDER_SEPARATOR; |
||||
} |
||||
|
||||
@Override |
||||
public boolean mkdir(String tenantCode, String path) throws IOException { |
||||
final String key = path + FOLDER_SEPARATOR; |
||||
if (!obsClient.doesObjectExist(bucketName, key)) { |
||||
createObsPrefix(bucketName, key); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
protected void createObsPrefix(final String bucketName, final String key) { |
||||
ObjectMetadata metadata = new ObjectMetadata(); |
||||
metadata.setContentLength(0L); |
||||
InputStream emptyContent = new ByteArrayInputStream(new byte[0]); |
||||
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, emptyContent); |
||||
obsClient.putObject(putObjectRequest); |
||||
} |
||||
|
||||
@Override |
||||
public String getResourceFullName(String tenantCode, String fileName) { |
||||
if (fileName.startsWith(FOLDER_SEPARATOR)) { |
||||
fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); |
||||
} |
||||
return String.format(FORMAT_S_S, getObsResDir(tenantCode), fileName); |
||||
} |
||||
|
||||
@Override |
||||
public String getResourceFileName(String tenantCode, String fullName) { |
||||
String resDir = getResDir(tenantCode); |
||||
return fullName.replaceFirst(resDir, ""); |
||||
} |
||||
|
||||
@Override |
||||
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { |
||||
if (fileName.startsWith(FOLDER_SEPARATOR)) { |
||||
fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); |
||||
} |
||||
return getDir(resourceType, tenantCode) + fileName; |
||||
} |
||||
|
||||
@Override |
||||
public boolean delete(String fullName, List<String> childrenPathList, boolean recursive) throws IOException { |
||||
// append the resource fullName to the list for deletion.
|
||||
childrenPathList.add(fullName); |
||||
|
||||
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName); |
||||
for (String deleteKys : childrenPathList) { |
||||
deleteObjectsRequest.addKeyAndVersion(deleteKys); |
||||
} |
||||
|
||||
try { |
||||
obsClient.deleteObjects(deleteObjectsRequest); |
||||
} catch (Exception e) { |
||||
log.error("delete objects error", e); |
||||
return false; |
||||
} |
||||
|
||||
return true; |
||||
} |
||||
|
||||
@Override |
||||
public void download(String tenantCode, String srcFilePath, String dstFilePath, |
||||
boolean overwrite) throws IOException { |
||||
File dstFile = new File(dstFilePath); |
||||
if (dstFile.isDirectory()) { |
||||
Files.delete(dstFile.toPath()); |
||||
} else { |
||||
Files.createDirectories(dstFile.getParentFile().toPath()); |
||||
} |
||||
ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath); |
||||
try ( |
||||
InputStream obsInputStream = obsObject.getObjectContent(); |
||||
FileOutputStream fos = new FileOutputStream(dstFilePath)) { |
||||
byte[] readBuf = new byte[1024]; |
||||
int readLen; |
||||
while ((readLen = obsInputStream.read(readBuf)) > 0) { |
||||
fos.write(readBuf, 0, readLen); |
||||
} |
||||
} catch (ObsException e) { |
||||
throw new IOException(e); |
||||
} catch (FileNotFoundException e) { |
||||
log.error("cannot find the destination file {}", dstFilePath); |
||||
throw e; |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public boolean exists(String fileName) throws IOException { |
||||
return obsClient.doesObjectExist(bucketName, fileName); |
||||
} |
||||
|
||||
@Override |
||||
public boolean delete(String filePath, boolean recursive) throws IOException { |
||||
try { |
||||
obsClient.deleteObject(bucketName, filePath); |
||||
return true; |
||||
} catch (ObsException e) { |
||||
log.error("fail to delete the object, the resource path is {}", filePath, e); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { |
||||
obsClient.copyObject(bucketName, srcPath, bucketName, dstPath); |
||||
if (deleteSource) { |
||||
obsClient.deleteObject(bucketName, srcPath); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
@Override |
||||
public String getDir(ResourceType resourceType, String tenantCode) { |
||||
switch (resourceType) { |
||||
case UDF: |
||||
return getUdfDir(tenantCode); |
||||
case FILE: |
||||
return getResDir(tenantCode); |
||||
case ALL: |
||||
return getObsDataBasePath(); |
||||
default: |
||||
return ""; |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, |
||||
boolean overwrite) throws IOException { |
||||
try { |
||||
obsClient.putObject(bucketName, dstPath, new File(srcFile)); |
||||
if (deleteSource) { |
||||
Files.delete(Paths.get(srcFile)); |
||||
} |
||||
return true; |
||||
} catch (ObsException e) { |
||||
log.error("upload failed, the bucketName is {}, the filePath is {}", bucketName, dstPath, e); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public List<String> vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException { |
||||
if (StringUtils.isBlank(filePath)) { |
||||
log.error("file path:{} is empty", filePath); |
||||
return Collections.emptyList(); |
||||
} |
||||
ObsObject obsObject = obsClient.getObject(bucketName, filePath); |
||||
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(obsObject.getObjectContent()))) { |
||||
Stream<String> stream = bufferedReader.lines().skip(skipLineNums).limit(limit); |
||||
return stream.collect(Collectors.toList()); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public ResUploadType returnStorageType() { |
||||
return ResUploadType.OBS; |
||||
} |
||||
|
||||
@Override |
||||
public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode, |
||||
ResourceType type) { |
||||
List<StorageEntity> storageEntityList = new ArrayList<>(); |
||||
LinkedList<StorageEntity> foldersToFetch = new LinkedList<>(); |
||||
|
||||
StorageEntity initialEntity = null; |
||||
try { |
||||
initialEntity = getFileStatus(path, defaultPath, tenantCode, type); |
||||
} catch (Exception e) { |
||||
log.error("error while listing files status recursively, path: {}", path, e); |
||||
return storageEntityList; |
||||
} |
||||
foldersToFetch.add(initialEntity); |
||||
|
||||
while (!foldersToFetch.isEmpty()) { |
||||
String pathToExplore = foldersToFetch.pop().getFullName(); |
||||
try { |
||||
List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); |
||||
for (StorageEntity temp : tempList) { |
||||
if (temp.isDirectory()) { |
||||
foldersToFetch.add(temp); |
||||
} |
||||
} |
||||
storageEntityList.addAll(tempList); |
||||
} catch (Exception e) { |
||||
log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e); |
||||
} |
||||
} |
||||
|
||||
return storageEntityList; |
||||
} |
||||
|
||||
@Override |
||||
public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode, |
||||
ResourceType type) throws Exception { |
||||
List<StorageEntity> storageEntityList = new ArrayList<>(); |
||||
|
||||
ListObjectsRequest request = new ListObjectsRequest(); |
||||
request.setBucketName(bucketName); |
||||
request.setPrefix(path); |
||||
request.setDelimiter(FOLDER_SEPARATOR); |
||||
ObjectListing result = null; |
||||
try { |
||||
result = obsClient.listObjects(request); |
||||
} catch (Exception e) { |
||||
throw new ServiceException("Get ObsClient file list exception", e); |
||||
} |
||||
|
||||
while (result != null) { |
||||
String nextMarker = result.getNextMarker(); |
||||
List<ObsObject> objects = result.getObjects(); |
||||
|
||||
for (ObsObject object : objects) { |
||||
if (!object.getObjectKey().endsWith(FOLDER_SEPARATOR)) { |
||||
// the path is a file
|
||||
String[] aliasArr = object.getObjectKey().split(FOLDER_SEPARATOR); |
||||
String alias = aliasArr[aliasArr.length - 1]; |
||||
String fileName = StringUtils.difference(defaultPath, object.getObjectKey()); |
||||
|
||||
StorageEntity entity = new StorageEntity(); |
||||
ObjectMetadata metadata = object.getMetadata(); |
||||
entity.setAlias(alias); |
||||
entity.setFileName(fileName); |
||||
entity.setFullName(object.getObjectKey()); |
||||
entity.setDirectory(false); |
||||
entity.setUserName(tenantCode); |
||||
entity.setType(type); |
||||
entity.setSize(metadata.getContentLength()); |
||||
entity.setCreateTime(metadata.getLastModified()); |
||||
entity.setUpdateTime(metadata.getLastModified()); |
||||
entity.setPfullName(path); |
||||
|
||||
storageEntityList.add(entity); |
||||
} |
||||
} |
||||
|
||||
for (String commonPrefix : result.getCommonPrefixes()) { |
||||
// the paths in commonPrefix are directories
|
||||
String suffix = StringUtils.difference(path, commonPrefix); |
||||
String fileName = StringUtils.difference(defaultPath, commonPrefix); |
||||
|
||||
StorageEntity entity = new StorageEntity(); |
||||
entity.setAlias(suffix); |
||||
entity.setFileName(fileName); |
||||
entity.setFullName(commonPrefix); |
||||
entity.setDirectory(true); |
||||
entity.setUserName(tenantCode); |
||||
entity.setType(type); |
||||
entity.setSize(0); |
||||
entity.setCreateTime(null); |
||||
entity.setUpdateTime(null); |
||||
entity.setPfullName(path); |
||||
|
||||
storageEntityList.add(entity); |
||||
} |
||||
|
||||
if (!StringUtils.isNotBlank(nextMarker)) { |
||||
break; |
||||
} |
||||
|
||||
request.setMarker(nextMarker); |
||||
try { |
||||
result = obsClient.listObjects(request); |
||||
} catch (Exception e) { |
||||
throw new ServiceException("Get ObsClient file list exception", e); |
||||
} |
||||
} |
||||
return storageEntityList; |
||||
} |
||||
|
||||
@Override |
||||
public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, |
||||
ResourceType type) throws Exception { |
||||
|
||||
if (path.endsWith(FOLDER_SEPARATOR)) { |
||||
// the path is a directory that may or may not exist in ObsClient
|
||||
String alias = findDirAlias(path); |
||||
String fileName = StringUtils.difference(defaultPath, path); |
||||
|
||||
StorageEntity entity = new StorageEntity(); |
||||
entity.setAlias(alias); |
||||
entity.setFileName(fileName); |
||||
entity.setFullName(path); |
||||
entity.setDirectory(true); |
||||
entity.setUserName(tenantCode); |
||||
entity.setType(type); |
||||
entity.setSize(0); |
||||
|
||||
return entity; |
||||
|
||||
} else { |
||||
GetObjectRequest request = new GetObjectRequest(); |
||||
request.setBucketName(bucketName); |
||||
request.setObjectKey(path); |
||||
ObsObject object; |
||||
try { |
||||
object = obsClient.getObject(request); |
||||
} catch (Exception e) { |
||||
throw new ServiceException("Get ObsClient file list exception", e); |
||||
} |
||||
|
||||
String[] aliasArr = object.getObjectKey().split(FOLDER_SEPARATOR); |
||||
String alias = aliasArr[aliasArr.length - 1]; |
||||
String fileName = StringUtils.difference(defaultPath, object.getObjectKey()); |
||||
|
||||
StorageEntity entity = new StorageEntity(); |
||||
ObjectMetadata metadata = object.getMetadata(); |
||||
entity.setAlias(alias); |
||||
entity.setFileName(fileName); |
||||
entity.setFullName(object.getObjectKey()); |
||||
entity.setDirectory(false); |
||||
entity.setUserName(tenantCode); |
||||
entity.setType(type); |
||||
entity.setSize(metadata.getContentLength()); |
||||
entity.setCreateTime(metadata.getLastModified()); |
||||
entity.setUpdateTime(metadata.getLastModified()); |
||||
|
||||
return entity; |
||||
} |
||||
|
||||
} |
||||
|
||||
@Override |
||||
public void deleteTenant(String tenantCode) throws Exception { |
||||
deleteTenantCode(tenantCode); |
||||
} |
||||
|
||||
public String getObsResDir(String tenantCode) { |
||||
return String.format("%s/" + RESOURCE_TYPE_FILE, getObsTenantDir(tenantCode)); |
||||
} |
||||
|
||||
public String getObsUdfDir(String tenantCode) { |
||||
return String.format("%s/" + RESOURCE_TYPE_UDF, getObsTenantDir(tenantCode)); |
||||
} |
||||
|
||||
public String getObsTenantDir(String tenantCode) { |
||||
return String.format(FORMAT_S_S, getObsDataBasePath(), tenantCode); |
||||
} |
||||
|
||||
public String getObsDataBasePath() { |
||||
if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { |
||||
return ""; |
||||
} else { |
||||
return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, ""); |
||||
} |
||||
} |
||||
|
||||
protected void deleteTenantCode(String tenantCode) { |
||||
deleteDir(getResDir(tenantCode)); |
||||
deleteDir(getUdfDir(tenantCode)); |
||||
} |
||||
|
||||
public void ensureBucketSuccessfullyCreated(String bucketName) { |
||||
if (StringUtils.isBlank(bucketName)) { |
||||
throw new IllegalArgumentException("resource.alibaba.cloud.obs.bucket.name is empty"); |
||||
} |
||||
|
||||
boolean existsBucket = obsClient.headBucket(bucketName); |
||||
if (!existsBucket) { |
||||
throw new IllegalArgumentException( |
||||
"bucketName: " + bucketName + " is not exists, you need to create them by yourself"); |
||||
} |
||||
|
||||
log.info("bucketName: {} has been found", bucketName); |
||||
} |
||||
|
||||
protected void deleteDir(String directoryName) { |
||||
if (obsClient.doesObjectExist(bucketName, directoryName)) { |
||||
obsClient.deleteObject(bucketName, directoryName); |
||||
} |
||||
} |
||||
|
||||
protected ObsClient buildObsClient() { |
||||
return new ObsClient(accessKeyId, accessKeySecret, endPoint); |
||||
} |
||||
|
||||
private String findDirAlias(String dirPath) { |
||||
if (!dirPath.endsWith(FOLDER_SEPARATOR)) { |
||||
return dirPath; |
||||
} |
||||
|
||||
Path path = Paths.get(dirPath); |
||||
return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR; |
||||
} |
||||
} |
||||
@ -0,0 +1,43 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.storage.obs; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; |
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; |
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageType; |
||||
|
||||
import com.google.auto.service.AutoService; |
||||
|
||||
@AutoService({StorageOperateFactory.class}) |
||||
public class ObsStorageOperatorFactory implements StorageOperateFactory { |
||||
|
||||
public ObsStorageOperatorFactory() { |
||||
} |
||||
|
||||
@Override |
||||
public StorageOperate createStorageOperate() { |
||||
ObsStorageOperator ossOperator = new ObsStorageOperator(); |
||||
ossOperator.init(); |
||||
return ossOperator; |
||||
} |
||||
|
||||
@Override |
||||
public StorageType getStorageOperate() { |
||||
return StorageType.OBS; |
||||
} |
||||
} |
||||
@ -0,0 +1,291 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.storage.obs; |
||||
|
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; |
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; |
||||
import static org.mockito.ArgumentMatchers.any; |
||||
import static org.mockito.ArgumentMatchers.anyString; |
||||
import static org.mockito.Mockito.doNothing; |
||||
import static org.mockito.Mockito.doReturn; |
||||
import static org.mockito.Mockito.spy; |
||||
import static org.mockito.Mockito.times; |
||||
import static org.mockito.Mockito.verify; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; |
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType; |
||||
|
||||
import java.io.IOException; |
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.BeforeEach; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
import com.obs.services.ObsClient; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
public class ObsStorageOperatorTest { |
||||
|
||||
private static final String ACCESS_KEY_ID_MOCK = "ACCESS_KEY_ID_MOCK"; |
||||
private static final String ACCESS_KEY_SECRET_MOCK = "ACCESS_KEY_SECRET_MOCK"; |
||||
private static final String END_POINT_MOCK = "END_POINT_MOCK"; |
||||
private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK"; |
||||
private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; |
||||
private static final String DIR_MOCK = "DIR_MOCK"; |
||||
private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; |
||||
private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; |
||||
private static final String FULL_NAME = "/tmp/dir1/"; |
||||
|
||||
private static final String DEFAULT_PATH = "/tmp/"; |
||||
@Mock |
||||
private ObsClient obsClientMock; |
||||
|
||||
private ObsStorageOperator obsOperator; |
||||
|
||||
@BeforeEach |
||||
public void setUp() throws Exception { |
||||
obsOperator = spy(new ObsStorageOperator()); |
||||
doReturn(ACCESS_KEY_ID_MOCK).when(obsOperator) |
||||
.readObsAccessKeyID(); |
||||
doReturn(ACCESS_KEY_SECRET_MOCK).when(obsOperator) |
||||
.readObsAccessKeySecret(); |
||||
doReturn(BUCKET_NAME_MOCK).when(obsOperator).readObsBucketName(); |
||||
doReturn(END_POINT_MOCK).when(obsOperator).readObsEndPoint(); |
||||
doReturn(obsClientMock).when(obsOperator).buildObsClient(); |
||||
doNothing().when(obsOperator).ensureBucketSuccessfullyCreated(any()); |
||||
|
||||
obsOperator.init(); |
||||
|
||||
} |
||||
|
||||
@Test |
||||
public void initObsOperator() { |
||||
verify(obsOperator, times(1)).buildObsClient(); |
||||
Assertions.assertEquals(ACCESS_KEY_ID_MOCK, obsOperator.getAccessKeyId()); |
||||
Assertions.assertEquals(ACCESS_KEY_SECRET_MOCK, obsOperator.getAccessKeySecret()); |
||||
Assertions.assertEquals(BUCKET_NAME_MOCK, obsOperator.getBucketName()); |
||||
} |
||||
|
||||
@Test |
||||
public void tearDownObsOperator() throws IOException { |
||||
doNothing().when(obsClientMock).close(); |
||||
obsOperator.close(); |
||||
verify(obsClientMock, times(1)).close(); |
||||
} |
||||
|
||||
@Test |
||||
public void createTenantResAndUdfDir() throws Exception { |
||||
doReturn(DIR_MOCK).when(obsOperator).getObsResDir(TENANT_CODE_MOCK); |
||||
doReturn(DIR_MOCK).when(obsOperator).getObsUdfDir(TENANT_CODE_MOCK); |
||||
doReturn(true).when(obsOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
obsOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); |
||||
verify(obsOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
} |
||||
|
||||
@Test |
||||
public void getResDir() { |
||||
final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); |
||||
final String dir = obsOperator.getResDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedResourceDir, dir); |
||||
} |
||||
|
||||
@Test |
||||
public void getUdfDir() { |
||||
final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); |
||||
final String dir = obsOperator.getUdfDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedUdfDir, dir); |
||||
} |
||||
|
||||
@Test |
||||
public void mkdirWhenDirExists() { |
||||
boolean isSuccess = false; |
||||
try { |
||||
final String key = DIR_MOCK + FOLDER_SEPARATOR; |
||||
doReturn(true).when(obsClientMock).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
isSuccess = obsOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
verify(obsClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
|
||||
} catch (IOException e) { |
||||
Assertions.fail("test failed due to unexpected IO exception"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isSuccess); |
||||
} |
||||
|
||||
@Test |
||||
public void mkdirWhenDirNotExists() { |
||||
boolean isSuccess = true; |
||||
try { |
||||
final String key = DIR_MOCK + FOLDER_SEPARATOR; |
||||
doReturn(false).when(obsClientMock).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
doNothing().when(obsOperator).createObsPrefix(BUCKET_NAME_MOCK, key); |
||||
isSuccess = obsOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
verify(obsClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
verify(obsOperator, times(1)).createObsPrefix(BUCKET_NAME_MOCK, key); |
||||
|
||||
} catch (IOException e) { |
||||
Assertions.fail("test failed due to unexpected IO exception"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isSuccess); |
||||
} |
||||
|
||||
@Test |
||||
public void getResourceFullName() { |
||||
final String expectedResourceFullName = |
||||
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
final String resourceFullName = obsOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
Assertions.assertEquals(expectedResourceFullName, resourceFullName); |
||||
} |
||||
|
||||
@Test |
||||
public void getResourceFileName() { |
||||
final String expectedResourceFileName = FILE_NAME_MOCK; |
||||
final String resourceFullName = |
||||
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
final String resourceFileName = obsOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName); |
||||
Assertions.assertEquals(expectedResourceFileName, resourceFileName); |
||||
} |
||||
|
||||
@Test |
||||
public void getFileName() { |
||||
final String expectedFileName = |
||||
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
final String fileName = obsOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
Assertions.assertEquals(expectedFileName, fileName); |
||||
} |
||||
|
||||
@Test |
||||
public void exists() { |
||||
boolean doesExist = false; |
||||
doReturn(true).when(obsClientMock).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); |
||||
try { |
||||
doesExist = obsOperator.exists(FILE_NAME_MOCK); |
||||
} catch (IOException e) { |
||||
Assertions.fail("unexpected IO exception in unit test"); |
||||
} |
||||
|
||||
Assertions.assertTrue(doesExist); |
||||
verify(obsClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); |
||||
} |
||||
|
||||
@Test |
||||
public void delete() { |
||||
boolean isDeleted = false; |
||||
doReturn(null).when(obsClientMock).deleteObject(anyString(), anyString()); |
||||
try { |
||||
isDeleted = obsOperator.delete(FILE_NAME_MOCK, true); |
||||
} catch (IOException e) { |
||||
Assertions.fail("unexpected IO exception in unit test"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isDeleted); |
||||
verify(obsClientMock, times(1)).deleteObject(anyString(), anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void copy() { |
||||
boolean isSuccess = false; |
||||
doReturn(null).when(obsClientMock).copyObject(anyString(), anyString(), anyString(), anyString()); |
||||
try { |
||||
isSuccess = obsOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); |
||||
} catch (IOException e) { |
||||
Assertions.fail("unexpected IO exception in unit test"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isSuccess); |
||||
verify(obsClientMock, times(1)).copyObject(anyString(), anyString(), anyString(), anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void deleteTenant() { |
||||
doNothing().when(obsOperator).deleteTenantCode(anyString()); |
||||
try { |
||||
obsOperator.deleteTenant(TENANT_CODE_MOCK); |
||||
} catch (Exception e) { |
||||
Assertions.fail("unexpected exception caught in unit test"); |
||||
} |
||||
|
||||
verify(obsOperator, times(1)).deleteTenantCode(anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void getObsResDir() { |
||||
final String expectedObsResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); |
||||
final String obsResDir = obsOperator.getObsResDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedObsResDir, obsResDir); |
||||
} |
||||
|
||||
@Test |
||||
public void getObsUdfDir() { |
||||
final String expectedObsUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); |
||||
final String obsUdfDir = obsOperator.getObsUdfDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedObsUdfDir, obsUdfDir); |
||||
} |
||||
|
||||
@Test |
||||
public void getObsTenantDir() { |
||||
final String expectedObsTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); |
||||
doReturn(DIR_MOCK).when(obsOperator).getObsDataBasePath(); |
||||
final String obsTenantDir = obsOperator.getObsTenantDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedObsTenantDir, obsTenantDir); |
||||
} |
||||
|
||||
@Test |
||||
public void deleteDir() { |
||||
doReturn(true).when(obsClientMock).doesObjectExist(anyString(), anyString()); |
||||
obsOperator.deleteDir(DIR_MOCK); |
||||
verify(obsClientMock, times(1)).deleteObject(anyString(), anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetFileStatus() throws Exception { |
||||
StorageEntity entity = obsOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); |
||||
Assertions.assertEquals(FULL_NAME, entity.getFullName()); |
||||
Assertions.assertEquals("dir1/", entity.getFileName()); |
||||
} |
||||
|
||||
@Test |
||||
public void testListFilesStatus() throws Exception { |
||||
List<StorageEntity> result = |
||||
obsOperator.listFilesStatus("dolphinscheduler/default/resources/", |
||||
"dolphinscheduler/default/resources/", |
||||
"default", ResourceType.FILE); |
||||
Assertions.assertEquals(0, result.size()); |
||||
} |
||||
|
||||
@Test |
||||
public void testListFilesStatusRecursively() throws Exception { |
||||
StorageEntity entity = new StorageEntity(); |
||||
entity.setFullName(FULL_NAME); |
||||
|
||||
doReturn(entity).when(obsOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); |
||||
doReturn(Collections.EMPTY_LIST).when(obsOperator).listFilesStatus(anyString(), anyString(), anyString(), |
||||
Mockito.any(ResourceType.class)); |
||||
|
||||
List<StorageEntity> result = |
||||
obsOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); |
||||
Assertions.assertEquals(0, result.size()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue