Rick Cheng
1 year ago
committed by
GitHub
3 changed files with 377 additions and 42 deletions
@ -0,0 +1,298 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
package org.apache.dolphinscheduler.plugin.storage.s3; |
||||
|
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; |
||||
import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; |
||||
import static org.mockito.ArgumentMatchers.anyString; |
||||
import static org.mockito.Mockito.doNothing; |
||||
import static org.mockito.Mockito.doReturn; |
||||
import static org.mockito.Mockito.times; |
||||
import static org.mockito.Mockito.verify; |
||||
|
||||
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; |
||||
import org.apache.dolphinscheduler.spi.enums.ResourceType; |
||||
|
||||
import java.io.IOException; |
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
|
||||
import org.junit.jupiter.api.Assertions; |
||||
import org.junit.jupiter.api.BeforeEach; |
||||
import org.junit.jupiter.api.Test; |
||||
import org.junit.jupiter.api.extension.ExtendWith; |
||||
import org.mockito.Mock; |
||||
import org.mockito.Mockito; |
||||
import org.mockito.junit.jupiter.MockitoExtension; |
||||
|
||||
import com.amazonaws.services.s3.AmazonS3; |
||||
import com.amazonaws.services.s3.model.ListObjectsV2Request; |
||||
import com.amazonaws.services.s3.model.ListObjectsV2Result; |
||||
|
||||
@ExtendWith(MockitoExtension.class) |
||||
public class S3StorageOperatorTest { |
||||
|
||||
private static final String ACCESS_KEY_ID_MOCK = "ACCESS_KEY_ID_MOCK"; |
||||
|
||||
private static final String ACCESS_KEY_SECRET_MOCK = "ACCESS_KEY_SECRET_MOCK"; |
||||
|
||||
private static final String REGION_MOCK = "REGION_MOCK"; |
||||
|
||||
private static final String END_POINT_MOCK = "END_POINT_MOCK"; |
||||
|
||||
private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK"; |
||||
|
||||
private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; |
||||
|
||||
private static final String DIR_MOCK = "DIR_MOCK"; |
||||
|
||||
private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; |
||||
|
||||
private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; |
||||
|
||||
private static final String FULL_NAME = "/tmp/dir1/"; |
||||
|
||||
private static final String DEFAULT_PATH = "/tmp/"; |
||||
|
||||
@Mock |
||||
private AmazonS3 s3Client; |
||||
|
||||
private S3StorageOperator s3StorageOperator; |
||||
|
||||
@BeforeEach |
||||
public void setUp() throws Exception { |
||||
s3StorageOperator = Mockito.spy(new S3StorageOperator()); |
||||
|
||||
doReturn(ACCESS_KEY_ID_MOCK).when(s3StorageOperator) |
||||
.readAccessKeyID(); |
||||
doReturn(ACCESS_KEY_SECRET_MOCK).when(s3StorageOperator) |
||||
.readAccessKeySecret(); |
||||
doReturn(REGION_MOCK).when(s3StorageOperator).readRegion(); |
||||
doReturn(BUCKET_NAME_MOCK).when(s3StorageOperator).readBucketName(); |
||||
doReturn(END_POINT_MOCK).when(s3StorageOperator).readEndPoint(); |
||||
Mockito.doReturn(s3Client) |
||||
.when(s3StorageOperator).buildS3Client(); |
||||
Mockito.doNothing() |
||||
.when(s3StorageOperator).checkBucketNameExists(Mockito.any()); |
||||
|
||||
s3StorageOperator.init(); |
||||
} |
||||
|
||||
@Test |
||||
public void testInit() { |
||||
verify(s3StorageOperator, times(1)).buildS3Client(); |
||||
Assertions.assertEquals(ACCESS_KEY_ID_MOCK, s3StorageOperator.getAccessKeyId()); |
||||
Assertions.assertEquals(ACCESS_KEY_SECRET_MOCK, s3StorageOperator.getAccessKeySecret()); |
||||
Assertions.assertEquals(REGION_MOCK, s3StorageOperator.getRegion()); |
||||
Assertions.assertEquals(BUCKET_NAME_MOCK, s3StorageOperator.getBucketName()); |
||||
} |
||||
|
||||
@Test |
||||
public void testTearDown() throws IOException { |
||||
doNothing().when(s3Client).shutdown(); |
||||
s3StorageOperator.close(); |
||||
verify(s3Client, times(1)).shutdown(); |
||||
} |
||||
|
||||
@Test |
||||
public void testCreateTenantResAndUdfDir() throws Exception { |
||||
doReturn(DIR_MOCK).when(s3StorageOperator).getS3ResDir(TENANT_CODE_MOCK); |
||||
doReturn(DIR_MOCK).when(s3StorageOperator).getS3UdfDir(TENANT_CODE_MOCK); |
||||
doReturn(true).when(s3StorageOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
s3StorageOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); |
||||
verify(s3StorageOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetResDir() { |
||||
final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); |
||||
final String dir = s3StorageOperator.getResDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedResourceDir, dir); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetUdfDir() { |
||||
final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); |
||||
final String dir = s3StorageOperator.getUdfDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedUdfDir, dir); |
||||
} |
||||
|
||||
@Test |
||||
public void mkdirWhenDirExists() { |
||||
boolean isSuccess = false; |
||||
try { |
||||
final String key = DIR_MOCK + FOLDER_SEPARATOR; |
||||
doReturn(true).when(s3Client).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
isSuccess = s3StorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
verify(s3Client, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
|
||||
} catch (IOException e) { |
||||
Assertions.fail("test failed due to unexpected IO exception"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isSuccess); |
||||
} |
||||
|
||||
@Test |
||||
public void mkdirWhenDirNotExists() { |
||||
boolean isSuccess = true; |
||||
try { |
||||
final String key = DIR_MOCK + FOLDER_SEPARATOR; |
||||
doReturn(false).when(s3Client).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
isSuccess = s3StorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); |
||||
verify(s3Client, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); |
||||
|
||||
} catch (IOException e) { |
||||
Assertions.fail("test failed due to unexpected IO exception"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isSuccess); |
||||
} |
||||
|
||||
@Test |
||||
public void getResourceFileName() { |
||||
final String expectedResourceFileName = |
||||
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
final String resourceFileName = s3StorageOperator.getResourceFileName(TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
Assertions.assertEquals(expectedResourceFileName, resourceFileName); |
||||
} |
||||
|
||||
@Test |
||||
public void getFileName() { |
||||
final String expectedFileName = |
||||
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
final String fileName = s3StorageOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); |
||||
Assertions.assertEquals(expectedFileName, fileName); |
||||
} |
||||
|
||||
@Test |
||||
public void exists() { |
||||
boolean doesExist = false; |
||||
doReturn(true).when(s3Client).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); |
||||
try { |
||||
doesExist = s3StorageOperator.exists(FILE_NAME_MOCK); |
||||
} catch (IOException e) { |
||||
Assertions.fail("unexpected IO exception in unit test"); |
||||
} |
||||
|
||||
Assertions.assertTrue(doesExist); |
||||
verify(s3Client, times(1)).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); |
||||
} |
||||
|
||||
@Test |
||||
public void delete() { |
||||
doNothing().when(s3Client).deleteObject(anyString(), anyString()); |
||||
try { |
||||
s3StorageOperator.delete(FILE_NAME_MOCK, true); |
||||
} catch (IOException e) { |
||||
Assertions.fail("unexpected IO exception in unit test"); |
||||
} |
||||
|
||||
verify(s3Client, times(1)).deleteObject(anyString(), anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void copy() { |
||||
boolean isSuccess = false; |
||||
doReturn(null).when(s3Client).copyObject(anyString(), anyString(), anyString(), anyString()); |
||||
doNothing().when(s3Client).deleteObject(anyString(), anyString()); |
||||
try { |
||||
isSuccess = s3StorageOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); |
||||
} catch (IOException e) { |
||||
Assertions.fail("unexpected IO exception in unit test"); |
||||
} |
||||
|
||||
Assertions.assertTrue(isSuccess); |
||||
verify(s3Client, times(1)).copyObject(anyString(), anyString(), anyString(), anyString()); |
||||
verify(s3Client, times(1)).deleteObject(anyString(), anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void deleteTenant() { |
||||
doNothing().when(s3StorageOperator).deleteTenantCode(anyString()); |
||||
try { |
||||
s3StorageOperator.deleteTenant(TENANT_CODE_MOCK); |
||||
} catch (Exception e) { |
||||
Assertions.fail("unexpected exception caught in unit test"); |
||||
} |
||||
|
||||
verify(s3StorageOperator, times(1)).deleteTenantCode(anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetS3ResDir() { |
||||
final String expectedS3ResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); |
||||
final String s3ResDir = s3StorageOperator.getS3ResDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedS3ResDir, s3ResDir); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetS3UdfDir() { |
||||
final String expectedS3UdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); |
||||
final String s3UdfDir = s3StorageOperator.getS3UdfDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedS3UdfDir, s3UdfDir); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetS3TenantDir() { |
||||
final String expectedS3TenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); |
||||
doReturn(DIR_MOCK).when(s3StorageOperator).getS3DataBasePath(); |
||||
final String s3TenantDir = s3StorageOperator.getS3TenantDir(TENANT_CODE_MOCK); |
||||
Assertions.assertEquals(expectedS3TenantDir, s3TenantDir); |
||||
} |
||||
|
||||
@Test |
||||
public void deleteDir() { |
||||
doReturn(true).when(s3Client).doesObjectExist(anyString(), anyString()); |
||||
s3StorageOperator.deleteDir(DIR_MOCK); |
||||
verify(s3Client, times(1)).deleteObject(anyString(), anyString()); |
||||
} |
||||
|
||||
@Test |
||||
public void testGetFileStatus() throws Exception { |
||||
doReturn(new ListObjectsV2Result()).when(s3Client).listObjectsV2(Mockito.any(ListObjectsV2Request.class)); |
||||
StorageEntity entity = |
||||
s3StorageOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); |
||||
Assertions.assertEquals(FULL_NAME, entity.getFullName()); |
||||
Assertions.assertEquals("dir1/", entity.getFileName()); |
||||
} |
||||
|
||||
@Test |
||||
public void testListFilesStatus() throws Exception { |
||||
doReturn(new ListObjectsV2Result()).when(s3Client).listObjectsV2(Mockito.any(ListObjectsV2Request.class)); |
||||
List<StorageEntity> result = |
||||
s3StorageOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); |
||||
Assertions.assertEquals(0, result.size()); |
||||
} |
||||
|
||||
@Test |
||||
public void testListFilesStatusRecursively() throws Exception { |
||||
StorageEntity entity = new StorageEntity(); |
||||
entity.setFullName(FULL_NAME); |
||||
|
||||
doReturn(entity).when(s3StorageOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, |
||||
ResourceType.FILE); |
||||
doReturn(Collections.EMPTY_LIST).when(s3StorageOperator).listFilesStatus(anyString(), anyString(), anyString(), |
||||
Mockito.any(ResourceType.class)); |
||||
|
||||
List<StorageEntity> result = |
||||
s3StorageOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, |
||||
ResourceType.FILE); |
||||
Assertions.assertEquals(0, result.size()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue