|
|
|
/*
|
|
|
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
* contributor license agreements. See the NOTICE file distributed with
|
|
|
|
* this work for additional information regarding copyright ownership.
|
|
|
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
* (the "License"); you may not use this file except in compliance with
|
|
|
|
* the License. You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package org.apache.dolphinscheduler.api.service;
|
|
|
|
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
|
|
import static org.mockito.ArgumentMatchers.eq;
|
|
|
|
import static org.mockito.Mockito.when;
|
|
|
|
|
|
|
|
import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
|
|
|
|
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
|
|
|
|
import org.apache.dolphinscheduler.api.enums.Status;
|
|
|
|
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
|
|
|
|
import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
|
|
|
|
import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
|
|
|
|
import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
|
|
|
|
import org.apache.dolphinscheduler.api.utils.PageInfo;
|
|
|
|
import org.apache.dolphinscheduler.api.utils.Result;
|
|
|
|
import org.apache.dolphinscheduler.common.constants.Constants;
|
|
|
|
import org.apache.dolphinscheduler.common.enums.UserType;
|
|
|
|
import org.apache.dolphinscheduler.common.utils.FileUtils;
|
|
|
|
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
|
|
|
|
import org.apache.dolphinscheduler.dao.entity.Tenant;
|
|
|
|
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
|
|
|
|
import org.apache.dolphinscheduler.dao.entity.User;
|
|
|
|
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
|
|
|
|
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
|
|
|
|
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
|
|
|
|
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
|
|
|
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
|
|
|
|
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
|
|
|
|
import org.apache.dolphinscheduler.spi.enums.ResourceType;
|
|
|
|
|
|
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
|
|
|
|
|
|
import java.io.IOException;
|
|
|
|
import java.nio.file.Path;
|
|
|
|
import java.nio.file.Paths;
|
|
|
|
import java.time.LocalDateTime;
|
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.Arrays;
|
|
|
|
import java.util.Collections;
|
|
|
|
import java.util.HashMap;
|
|
|
|
import java.util.HashSet;
|
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
|
|
|
import java.util.Random;
|
|
|
|
import java.util.Set;
|
|
|
|
|
|
|
|
import org.junit.jupiter.api.AfterEach;
|
|
|
|
import org.junit.jupiter.api.Assertions;
|
|
|
|
import org.junit.jupiter.api.BeforeEach;
|
|
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
import org.junit.jupiter.api.extension.ExtendWith;
|
|
|
|
import org.mockito.InjectMocks;
|
|
|
|
import org.mockito.Mock;
|
|
|
|
import org.mockito.MockedStatic;
|
|
|
|
import org.mockito.Mockito;
|
|
|
|
import org.mockito.junit.jupiter.MockitoExtension;
|
|
|
|
import org.mockito.junit.jupiter.MockitoSettings;
|
|
|
|
import org.mockito.quality.Strictness;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.springframework.mock.web.MockMultipartFile;
|
|
|
|
|
|
|
|
import com.google.common.io.Files;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* resources service test
|
|
|
|
*/
|
|
|
|
@ExtendWith(MockitoExtension.class)
|
|
|
|
@MockitoSettings(strictness = Strictness.LENIENT)
|
|
|
|
public class ResourcesServiceTest {
|
|
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class);
|
|
|
|
|
|
|
|
@InjectMocks
|
|
|
|
private ResourcesServiceImpl resourcesService;
|
|
|
|
|
|
|
|
@Mock
|
|
|
|
private TenantMapper tenantMapper;
|
|
|
|
|
|
|
|
@Mock
|
[Feature-8612][RESOURCE] extend s3 to the storage of ds (#8637)
* feat(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.extend the type of storage
3.add the s3utils
to manager resource
4.automatic inject the storage in addition to your
config
* fix(resource manager): update the dependency
* fix(resource manager): extend s3 to the storage of ds
fix the constant of hadooputils
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager):
merge the unitTest:
1.TenantServiceImpl
2.ResourceServiceImpl
3.UserServiceImpl
* fix(resource manager): extend s3 to the storage of ds
merge the resourceServiceTest
* fix(resource manager): test cancel the test method
createTenant verifyTenant
* fix(resource manager): merge the code follow the check-result of sonar
* fix(resource manager): extend s3 to the storage of ds
fit the spell question
* fix(resource manager): extend s3 to the storage of ds
revert the common.properties
* fix(resource manager): extend s3 to the storage of ds
update the storageConfig with None
* fix(resource manager): extend s3 to the storage of ds
fix the judge of resourceType
* fix(resource manager): extend s3 to the storage of ds
undo the compile-mysql
* fix(resource manager): extend s3 to the storage of ds
delete hadoop aws
* fix(resource manager): extend s3 to the storage of ds
update the know-dependencies to delete aws 1.7.4
update the e2e
file-manager common.properties
* fix(resource manager): extend s3 to the storage of ds
update the aws-region
* fix(resource manager): extend s3 to the storage of ds
fix the storageconfig init
* fix(resource manager): update e2e docker-compose
update e2e docker-compose
* fix(resource manager): extend s3 to the storage of ds
revent the e2e common.proprites
print the resource type in propertyUtil
* fix(resource manager): extend s3 to the storage of ds
1.println the properties
* fix(resource manager): println the s3 info
* fix(resource manager): extend s3 to the storage of ds
delete the info and upgrade the s3 info to e2e
* fix(resource manager): extend s3 to the storage of ds
add the bucket init
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): extend s3 to the storage of ds
upgrade the s3 endpoint
* fix(resource manager): withPathStyleAccessEnabled(true)
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): upgrade the s3client builder
* fix(resource manager): correct the s3 point to s3client
* fix(resource manager): update the constant BUCKET_NAME
* fix(resource manager): e2e s3 endpoint -> s3:9000
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* style(resource manager): add info to createBucket
* style(resource manager): debug the log
* ci(resource manager): test
test s3
* ci(ci): add INSERT INTO dolphinscheduler.t_ds_tenant (id, tenant_code, description, queue_id, create_time, update_time) VALUES(1, 'root', NULL, 1, NULL, NULL); to h2.sql
* fix(resource manager): update the h2 sql
* fix(resource manager): solve to delete the tenant
* style(resource manager): merge the style end delete the unuse s3 config
* fix(resource manager): extend s3 to the storage of ds
UPDATE the rename resources when s3
* fix(resource manager): extend s3 to the storage of ds
1.fix the code style of QuartzImpl
* fix(resource manager): extend s3 to the storage of ds
1.impoort restore_type to CommonUtils
* fix(resource manager): update the work thread
* fix(resource manager): update the baseTaskProcessor
* fix(resource manager): upgrade dolphinscheduler-standalone-server.xml
* fix(resource manager): add user Info to dolphinscheduler_h2.sql
* fix(resource manager): merge the resourceType to NONE
* style(upgrade the log level to info):
* fix(resource manager): sysnc the h2.sql
* fix(resource manager): update the merge the user tenant
* fix(resource manager): merge the resourcesServiceImpl
* fix(resource manager):
when the storage is s3 ,that the directory can't be renamed
* fix(resource manager): in s3 ,the directory cannot be renamed
* fix(resource manager): delete the deleteRenameDirectory in E2E
* fix(resource manager): check the style and recoverd the test
* fix(resource manager): delete the log.print(LoginUser)
2 years ago
|
|
|
private StorageOperate storageOperate;
|
|
|
|
|
|
|
|
@Mock
|
|
|
|
private UserMapper userMapper;
|
|
|
|
|
|
|
|
@Mock
|
|
|
|
private UdfFuncMapper udfFunctionMapper;
|
|
|
|
|
|
|
|
@Mock
|
|
|
|
private ProcessDefinitionMapper processDefinitionMapper;
|
|
|
|
|
|
|
|
@Mock
|
|
|
|
private ResourcePermissionCheckService resourcePermissionCheckService;
|
|
|
|
|
|
|
|
private static final Logger serviceLogger = LoggerFactory.getLogger(BaseServiceImpl.class);
|
|
|
|
|
|
|
|
private static final Logger resourceLogger = LoggerFactory.getLogger(ResourcesServiceImpl.class);
|
|
|
|
|
|
|
|
private MockedStatic<FileUtils> mockedStaticFileUtils;
|
|
|
|
|
|
|
|
private MockedStatic<Files> mockedStaticFiles;
|
|
|
|
|
|
|
|
private MockedStatic<org.apache.dolphinscheduler.api.utils.FileUtils> mockedStaticDolphinschedulerFileUtils;
|
|
|
|
|
|
|
|
private MockedStatic<PropertyUtils> mockedStaticPropertyUtils;
|
|
|
|
|
|
|
|
private MockedStatic<Paths> mockedStaticPaths;
|
|
|
|
|
|
|
|
private MockedStatic<java.nio.file.Files> filesMockedStatic;
|
|
|
|
|
|
|
|
private Throwable exception;
|
|
|
|
|
|
|
|
@BeforeEach
|
|
|
|
public void setUp() {
|
|
|
|
mockedStaticFileUtils = Mockito.mockStatic(FileUtils.class);
|
|
|
|
mockedStaticFiles = Mockito.mockStatic(Files.class);
|
|
|
|
mockedStaticDolphinschedulerFileUtils =
|
|
|
|
Mockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class);
|
|
|
|
|
|
|
|
mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
|
|
|
|
mockedStaticPaths = Mockito.mockStatic(Paths.class);
|
|
|
|
filesMockedStatic = Mockito.mockStatic(java.nio.file.Files.class);
|
|
|
|
}
|
|
|
|
|
|
|
|
@AfterEach
|
|
|
|
public void after() {
|
|
|
|
mockedStaticFileUtils.close();
|
|
|
|
mockedStaticFiles.close();
|
|
|
|
mockedStaticDolphinschedulerFileUtils.close();
|
|
|
|
mockedStaticPropertyUtils.close();
|
|
|
|
mockedStaticPaths.close();
|
|
|
|
filesMockedStatic.close();
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testCreateResource() {
|
|
|
|
User user = new User();
|
|
|
|
user.setId(1);
|
|
|
|
user.setUserType(UserType.GENERAL_USER);
|
|
|
|
|
|
|
|
// CURRENT_LOGIN_USER_TENANT_NOT_EXIST
|
|
|
|
when(userMapper.selectById(user.getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(null);
|
|
|
|
Assertions.assertThrows(ServiceException.class,
|
|
|
|
() -> resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
|
|
|
|
new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()), "/"));
|
|
|
|
// set tenant for user
|
|
|
|
user.setTenantId(1);
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
|
|
|
|
// RESOURCE_FILE_IS_EMPTY
|
|
|
|
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
|
|
|
|
Result result = resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE,
|
|
|
|
mockMultipartFile, "/");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// RESOURCE_SUFFIX_FORBID_CHANGE
|
|
|
|
mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
|
|
|
|
when(Files.getFileExtension("test.pdf")).thenReturn("pdf");
|
|
|
|
when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
|
|
|
|
result = resourcesService.uploadResource(user, "ResourcesServiceTest.jar", ResourceType.FILE, mockMultipartFile,
|
|
|
|
"/");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// UDF_RESOURCE_SUFFIX_NOT_JAR
|
|
|
|
mockMultipartFile =
|
|
|
|
new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes());
|
|
|
|
when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf");
|
|
|
|
result = resourcesService.uploadResource(user, "ResourcesServiceTest.pdf", ResourceType.UDF, mockMultipartFile,
|
|
|
|
"/");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// FULL_FILE_NAME_TOO_LONG
|
|
|
|
String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf";
|
|
|
|
mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes());
|
|
|
|
when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf");
|
|
|
|
// '/databasePath/tenantCode/RESOURCE/'
|
|
|
|
when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
result = resourcesService.uploadResource(user, tooLongFileName, ResourceType.FILE, mockMultipartFile, "/");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testCreateDirecotry() {
|
|
|
|
User user = new User();
|
|
|
|
user.setId(1);
|
|
|
|
user.setUserType(UserType.GENERAL_USER);
|
|
|
|
|
|
|
|
// RESOURCE_EXIST
|
|
|
|
user.setId(1);
|
|
|
|
user.setTenantId(1);
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
when(userMapper.selectById(user.getId())).thenReturn(getUser());
|
|
|
|
when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
try {
|
|
|
|
when(storageOperate.exists("/dolphinscheduler/123/resources/directoryTest")).thenReturn(true);
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error(e.getMessage(), e);
|
|
|
|
}
|
|
|
|
Result result = resourcesService.createDirectory(user, "directoryTest", ResourceType.FILE, -1, "/");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testUpdateResource() {
|
|
|
|
User user = new User();
|
|
|
|
user.setId(1);
|
|
|
|
user.setUserType(UserType.GENERAL_USER);
|
|
|
|
user.setTenantId(1);
|
|
|
|
|
|
|
|
when(userMapper.selectById(user.getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
|
|
|
|
// USER_NO_OPERATION_PERM
|
|
|
|
user.setUserType(UserType.GENERAL_USER);
|
|
|
|
// tenant who have access to resource is 123,
|
|
|
|
Tenant tenantWNoPermission = new Tenant();
|
|
|
|
tenantWNoPermission.setTenantCode("321");
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission);
|
|
|
|
Result result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest",
|
|
|
|
"123", "ResourcesServiceTest", ResourceType.FILE, null);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
try {
|
|
|
|
when(storageOperate.exists(Mockito.any())).thenReturn(false);
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error(e.getMessage(), e);
|
|
|
|
}
|
|
|
|
|
|
|
|
try {
|
|
|
|
when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest",
|
|
|
|
"/dolphinscheduler/123/resources/", "123", ResourceType.FILE))
|
|
|
|
.thenReturn(getStorageEntityResource());
|
|
|
|
result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest",
|
|
|
|
"123", "ResourcesServiceTest", ResourceType.FILE, null);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest",
|
|
|
|
e);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Tests for udf resources.
|
|
|
|
// RESOURCE_EXIST
|
|
|
|
try {
|
|
|
|
when(storageOperate.exists("/dolphinscheduler/123/resources/ResourcesServiceTest2.jar")).thenReturn(true);
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error("error occurred when checking resource: "
|
|
|
|
+ "/dolphinscheduler/123/resources/ResourcesServiceTest2.jar");
|
|
|
|
}
|
|
|
|
|
|
|
|
try {
|
|
|
|
when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
|
|
|
|
"/dolphinscheduler/123/resources/", "123", ResourceType.UDF))
|
|
|
|
.thenReturn(getStorageEntityUdfResource());
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(e.getMessage() + " Resource path: {}",
|
|
|
|
"/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", e);
|
|
|
|
}
|
|
|
|
result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
|
|
|
|
"123", "ResourcesServiceTest2.jar", ResourceType.UDF, null);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// TENANT_NOT_EXIST
|
|
|
|
when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
|
|
|
|
Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResource(user,
|
|
|
|
"ResourcesServiceTest1.jar", "", "ResourcesServiceTest", ResourceType.UDF, null));
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
|
|
|
|
result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
|
|
|
|
"123", "ResourcesServiceTest1.jar", ResourceType.UDF, null);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testQueryResourceListPaging() {
|
|
|
|
User loginUser = new User();
|
|
|
|
loginUser.setId(1);
|
|
|
|
loginUser.setTenantId(1);
|
|
|
|
loginUser.setTenantCode("tenant1");
|
|
|
|
loginUser.setUserType(UserType.ADMIN_USER);
|
|
|
|
List<StorageEntity> mockResList = new ArrayList<StorageEntity>();
|
|
|
|
mockResList.add(getStorageEntityResource());
|
|
|
|
List<User> mockUserList = new ArrayList<User>();
|
|
|
|
mockUserList.add(getUser());
|
|
|
|
when(userMapper.selectList(null)).thenReturn(mockUserList);
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
|
|
|
|
when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
|
|
|
|
try {
|
|
|
|
when(storageOperate.listFilesStatus("/dolphinscheduler/123/resources/", "/dolphinscheduler/123/resources/",
|
|
|
|
"123", ResourceType.FILE)).thenReturn(mockResList);
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error("QueryResourceListPaging Error");
|
|
|
|
}
|
|
|
|
Result result = resourcesService.queryResourceListPaging(loginUser, "", "", ResourceType.FILE, "Test", 1, 10);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
|
|
|
|
PageInfo pageInfo = (PageInfo) result.getData();
|
|
|
|
Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList()));
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testQueryResourceList() {
|
|
|
|
User loginUser = new User();
|
|
|
|
loginUser.setId(0);
|
|
|
|
loginUser.setUserType(UserType.ADMIN_USER);
|
|
|
|
|
|
|
|
when(userMapper.selectList(null)).thenReturn(Arrays.asList(loginUser));
|
|
|
|
when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
|
|
|
|
when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
|
|
|
|
when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/resources/",
|
|
|
|
"/dolphinscheduler/123/resources/", "123", ResourceType.FILE))
|
|
|
|
.thenReturn(Arrays.asList(getStorageEntityResource()));
|
|
|
|
Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE, "");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
|
|
|
|
List<ResourceComponent> resourceList = (List<ResourceComponent>) result.get(Constants.DATA_LIST);
|
|
|
|
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
|
|
|
|
|
|
|
|
// test udf
|
|
|
|
when(storageOperate.getUdfDir("123")).thenReturn("/dolphinscheduler/123/udfs/");
|
|
|
|
when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/udfs/", "/dolphinscheduler/123/udfs/",
|
|
|
|
"123", ResourceType.UDF)).thenReturn(Arrays.asList(getStorageEntityUdfResource()));
|
|
|
|
loginUser.setUserType(UserType.GENERAL_USER);
|
|
|
|
result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, "");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
|
|
|
|
resourceList = (List<ResourceComponent>) result.get(Constants.DATA_LIST);
|
|
|
|
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testDelete() throws Exception {
|
|
|
|
|
|
|
|
User loginUser = new User();
|
|
|
|
loginUser.setId(0);
|
|
|
|
loginUser.setUserType(UserType.GENERAL_USER);
|
|
|
|
|
|
|
|
// TENANT_NOT_EXIST
|
|
|
|
loginUser.setUserType(UserType.ADMIN_USER);
|
|
|
|
loginUser.setTenantId(2);
|
|
|
|
when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
|
|
|
|
Assertions.assertThrows(ServiceException.class, () -> resourcesService.delete(loginUser, "", ""));
|
|
|
|
|
|
|
|
// RESOURCE_NOT_EXIST
|
|
|
|
when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
|
|
|
|
when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest", null, "123", null))
|
|
|
|
.thenReturn(getStorageEntityResource());
|
|
|
|
Result result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResNotExist", "123");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
loginUser.setTenantId(1);
|
|
|
|
result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResourcesServiceTest", "123");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testVerifyResourceName() {
|
|
|
|
|
|
|
|
User user = new User();
|
|
|
|
user.setId(1);
|
|
|
|
user.setUserType(UserType.GENERAL_USER);
|
|
|
|
try {
|
|
|
|
when(storageOperate.exists("/ResourcesServiceTest.jar")).thenReturn(true);
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error("error occurred when checking resource: /ResourcesServiceTest.jar\"");
|
|
|
|
}
|
|
|
|
Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// RESOURCE_FILE_EXIST
|
|
|
|
result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
|
|
|
|
logger.info(result.toString());
|
|
|
|
Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode());
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testReadResource() {
|
|
|
|
// RESOURCE_NOT_EXIST
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
|
|
|
|
Result result = resourcesService.readResource(getUser(), "", "", 1, 10);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
|
|
|
|
|
|
|
|
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
|
|
|
|
when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
|
|
|
|
result = resourcesService.readResource(getUser(), "", "", 1, 10);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// USER_NOT_EXIST
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(null);
|
|
|
|
when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
|
|
|
|
when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
|
|
|
|
result = resourcesService.readResource(getUser(), "", "", 1, 10);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode());
|
|
|
|
|
|
|
|
// TENANT_NOT_EXIST
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null);
|
|
|
|
Assertions.assertThrows(ServiceException.class, () -> resourcesService.readResource(getUser(), "", "", 1, 10));
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
|
|
|
|
try {
|
|
|
|
when(storageOperate.exists(Mockito.any())).thenReturn(true);
|
|
|
|
when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent());
|
|
|
|
} catch (IOException e) {
|
[Feature-8612][RESOURCE] extend s3 to the storage of ds (#8637)
* feat(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.extend the type of storage
3.add the s3utils
to manager resource
4.automatic inject the storage in addition to your
config
* fix(resource manager): update the dependency
* fix(resource manager): extend s3 to the storage of ds
fix the constant of hadooputils
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager):
merge the unitTest:
1.TenantServiceImpl
2.ResourceServiceImpl
3.UserServiceImpl
* fix(resource manager): extend s3 to the storage of ds
merge the resourceServiceTest
* fix(resource manager): test cancel the test method
createTenant verifyTenant
* fix(resource manager): merge the code follow the check-result of sonar
* fix(resource manager): extend s3 to the storage of ds
fit the spell question
* fix(resource manager): extend s3 to the storage of ds
revert the common.properties
* fix(resource manager): extend s3 to the storage of ds
update the storageConfig with None
* fix(resource manager): extend s3 to the storage of ds
fix the judge of resourceType
* fix(resource manager): extend s3 to the storage of ds
undo the compile-mysql
* fix(resource manager): extend s3 to the storage of ds
delete hadoop aws
* fix(resource manager): extend s3 to the storage of ds
update the know-dependencies to delete aws 1.7.4
update the e2e
file-manager common.properties
* fix(resource manager): extend s3 to the storage of ds
update the aws-region
* fix(resource manager): extend s3 to the storage of ds
fix the storageconfig init
* fix(resource manager): update e2e docker-compose
update e2e docker-compose
* fix(resource manager): extend s3 to the storage of ds
revent the e2e common.proprites
print the resource type in propertyUtil
* fix(resource manager): extend s3 to the storage of ds
1.println the properties
* fix(resource manager): println the s3 info
* fix(resource manager): extend s3 to the storage of ds
delete the info and upgrade the s3 info to e2e
* fix(resource manager): extend s3 to the storage of ds
add the bucket init
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): extend s3 to the storage of ds
upgrade the s3 endpoint
* fix(resource manager): withPathStyleAccessEnabled(true)
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): upgrade the s3client builder
* fix(resource manager): correct the s3 point to s3client
* fix(resource manager): update the constant BUCKET_NAME
* fix(resource manager): e2e s3 endpoint -> s3:9000
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* style(resource manager): add info to createBucket
* style(resource manager): debug the log
* ci(resource manager): test
test s3
* ci(ci): add INSERT INTO dolphinscheduler.t_ds_tenant (id, tenant_code, description, queue_id, create_time, update_time) VALUES(1, 'root', NULL, 1, NULL, NULL); to h2.sql
* fix(resource manager): update the h2 sql
* fix(resource manager): solve to delete the tenant
* style(resource manager): merge the style end delete the unuse s3 config
* fix(resource manager): extend s3 to the storage of ds
UPDATE the rename resources when s3
* fix(resource manager): extend s3 to the storage of ds
1.fix the code style of QuartzImpl
* fix(resource manager): extend s3 to the storage of ds
1.impoort restore_type to CommonUtils
* fix(resource manager): update the work thread
* fix(resource manager): update the baseTaskProcessor
* fix(resource manager): upgrade dolphinscheduler-standalone-server.xml
* fix(resource manager): add user Info to dolphinscheduler_h2.sql
* fix(resource manager): merge the resourceType to NONE
* style(upgrade the log level to info):
* fix(resource manager): sysnc the h2.sql
* fix(resource manager): update the merge the user tenant
* fix(resource manager): merge the resourcesServiceImpl
* fix(resource manager):
when the storage is s3 ,that the directory can't be renamed
* fix(resource manager): in s3 ,the directory cannot be renamed
* fix(resource manager): delete the deleteRenameDirectory in E2E
* fix(resource manager): check the style and recoverd the test
* fix(resource manager): delete the log.print(LoginUser)
2 years ago
|
|
|
logger.error("storage error", e);
|
|
|
|
}
|
|
|
|
when(Files.getFileExtension("test.jar")).thenReturn("jar");
|
|
|
|
result = resourcesService.readResource(getUser(), "test.jar", "", 1, 10);
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testCreateOrUpdateResource() throws Exception {
|
|
|
|
User user = getUser();
|
|
|
|
when(userMapper.queryByUserNameAccurately(user.getUserName())).thenReturn(getUser());
|
|
|
|
|
|
|
|
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
|
|
|
|
exception = Assertions.assertThrows(IllegalArgumentException.class,
|
|
|
|
() -> resourcesService.createOrUpdateResource(user.getUserName(), "filename", "my-content"));
|
|
|
|
Assertions.assertTrue(
|
|
|
|
exception.getMessage().contains("Not allow create or update resources without extension name"));
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
when(storageOperate.getResDir(user.getTenantCode())).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
|
|
|
|
when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
|
|
|
|
when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any()))
|
|
|
|
.thenReturn(getStorageEntityResource());
|
|
|
|
StorageEntity storageEntity =
|
|
|
|
resourcesService.createOrUpdateResource(user.getUserName(), "filename.txt", "my-content");
|
|
|
|
Assertions.assertNotNull(storageEntity);
|
|
|
|
assertEquals("/dolphinscheduler/123/resources/ResourcesServiceTest", storageEntity.getFullName());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testUpdateResourceContent() throws Exception {
|
|
|
|
// RESOURCE_PATH_ILLEGAL
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/tmp");
|
|
|
|
ServiceException serviceException =
|
|
|
|
Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(),
|
|
|
|
"/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "123", "content"));
|
|
|
|
assertTrue(serviceException.getMessage()
|
|
|
|
.contains("Resource file: /dolphinscheduler/123/resources/ResourcesServiceTest.jar is illegal"));
|
|
|
|
|
|
|
|
// RESOURCE_NOT_EXIST
|
|
|
|
when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/dolphinscheduler/123/resources");
|
|
|
|
when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "", "123",
|
|
|
|
ResourceType.FILE)).thenReturn(null);
|
|
|
|
Result result = resourcesService.updateResourceContent(getUser(),
|
|
|
|
"/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "123", "content");
|
|
|
|
assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
|
|
|
|
when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
|
|
|
|
when(storageOperate.getFileStatus("/dolphinscheduler/123/resources", "", "123", ResourceType.FILE))
|
|
|
|
.thenReturn(getStorageEntityResource());
|
|
|
|
|
|
|
|
result = resourcesService.updateResourceContent(getUser(), "/dolphinscheduler/123/resources", "123", "content");
|
|
|
|
assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
|
|
|
|
|
|
|
|
// USER_NOT_EXIST
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(null);
|
|
|
|
result = resourcesService.updateResourceContent(getUser(), "/dolphinscheduler/123/resources/123.class", "123",
|
|
|
|
"content");
|
|
|
|
Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
|
|
|
|
|
|
|
|
// TENANT_NOT_EXIST
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(null);
|
|
|
|
Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(),
|
|
|
|
"/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "123", "content"));
|
|
|
|
|
|
|
|
// SUCCESS
|
|
|
|
when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "", "123",
|
|
|
|
ResourceType.FILE)).thenReturn(getStorageEntityResource());
|
|
|
|
|
|
|
|
when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar");
|
|
|
|
when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
|
|
|
|
when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
|
|
|
|
when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
|
|
|
|
result = resourcesService.updateResourceContent(getUser(),
|
|
|
|
"/dolphinscheduler/123/resources/ResourcesServiceTest.jar", "123", "content");
|
|
|
|
logger.info(result.toString());
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testDownloadResource() {
|
|
|
|
when(tenantMapper.queryById(1)).thenReturn(getTenant());
|
|
|
|
when(userMapper.selectById(1)).thenReturn(getUser());
|
|
|
|
org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
|
|
|
|
Path path = Mockito.mock(Path.class);
|
|
|
|
when(Paths.get(Mockito.any())).thenReturn(path);
|
|
|
|
try {
|
|
|
|
when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L);
|
|
|
|
// resource null
|
|
|
|
org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
|
|
|
|
Assertions.assertNull(resource);
|
|
|
|
|
|
|
|
when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock);
|
|
|
|
resource = resourcesService.downloadResource(getUser(), "");
|
|
|
|
Assertions.assertNotNull(resource);
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error("DownloadResource error", e);
|
|
|
|
Assertions.assertTrue(false);
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testDeleteDataTransferData() throws Exception {
|
|
|
|
User user = getUser();
|
|
|
|
when(userMapper.selectById(user.getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
|
|
|
|
|
|
|
|
StorageEntity storageEntity1 = Mockito.mock(StorageEntity.class);
|
|
|
|
StorageEntity storageEntity2 = Mockito.mock(StorageEntity.class);
|
|
|
|
StorageEntity storageEntity3 = Mockito.mock(StorageEntity.class);
|
|
|
|
StorageEntity storageEntity4 = Mockito.mock(StorageEntity.class);
|
|
|
|
StorageEntity storageEntity5 = Mockito.mock(StorageEntity.class);
|
|
|
|
|
|
|
|
when(storageEntity1.getFullName()).thenReturn("DATA_TRANSFER/20220101");
|
|
|
|
when(storageEntity2.getFullName()).thenReturn("DATA_TRANSFER/20220102");
|
|
|
|
when(storageEntity3.getFullName()).thenReturn("DATA_TRANSFER/20220103");
|
|
|
|
when(storageEntity4.getFullName()).thenReturn("DATA_TRANSFER/20220104");
|
|
|
|
when(storageEntity5.getFullName()).thenReturn("DATA_TRANSFER/20220105");
|
|
|
|
|
|
|
|
List<StorageEntity> storageEntityList = new ArrayList<>();
|
|
|
|
storageEntityList.add(storageEntity1);
|
|
|
|
storageEntityList.add(storageEntity2);
|
|
|
|
storageEntityList.add(storageEntity3);
|
|
|
|
storageEntityList.add(storageEntity4);
|
|
|
|
storageEntityList.add(storageEntity5);
|
|
|
|
|
|
|
|
when(storageOperate.listFilesStatus(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()))
|
|
|
|
.thenReturn(storageEntityList);
|
|
|
|
|
|
|
|
LocalDateTime localDateTime = LocalDateTime.of(2022, 1, 5, 0, 0, 0);
|
|
|
|
try (MockedStatic<LocalDateTime> mockHook = Mockito.mockStatic(LocalDateTime.class)) {
|
|
|
|
mockHook.when(LocalDateTime::now).thenReturn(localDateTime);
|
|
|
|
DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 3);
|
|
|
|
|
|
|
|
assertEquals(response.getSuccessList().size(), 2);
|
|
|
|
assertEquals(response.getSuccessList().get(0), "DATA_TRANSFER/20220101");
|
|
|
|
assertEquals(response.getSuccessList().get(1), "DATA_TRANSFER/20220102");
|
|
|
|
}
|
|
|
|
|
|
|
|
try (MockedStatic<LocalDateTime> mockHook = Mockito.mockStatic(LocalDateTime.class)) {
|
|
|
|
mockHook.when(LocalDateTime::now).thenReturn(localDateTime);
|
|
|
|
DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 0);
|
|
|
|
assertEquals(response.getSuccessList().size(), 5);
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
public void testCatFile() {
|
|
|
|
// SUCCESS
|
|
|
|
try {
|
[Feature-8612][RESOURCE] extend s3 to the storage of ds (#8637)
* feat(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.extend the type of storage
3.add the s3utils
to manager resource
4.automatic inject the storage in addition to your
config
* fix(resource manager): update the dependency
* fix(resource manager): extend s3 to the storage of ds
fix the constant of hadooputils
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager):
merge the unitTest:
1.TenantServiceImpl
2.ResourceServiceImpl
3.UserServiceImpl
* fix(resource manager): extend s3 to the storage of ds
merge the resourceServiceTest
* fix(resource manager): test cancel the test method
createTenant verifyTenant
* fix(resource manager): merge the code follow the check-result of sonar
* fix(resource manager): extend s3 to the storage of ds
fit the spell question
* fix(resource manager): extend s3 to the storage of ds
revert the common.properties
* fix(resource manager): extend s3 to the storage of ds
update the storageConfig with None
* fix(resource manager): extend s3 to the storage of ds
fix the judge of resourceType
* fix(resource manager): extend s3 to the storage of ds
undo the compile-mysql
* fix(resource manager): extend s3 to the storage of ds
delete hadoop aws
* fix(resource manager): extend s3 to the storage of ds
update the know-dependencies to delete aws 1.7.4
update the e2e
file-manager common.properties
* fix(resource manager): extend s3 to the storage of ds
update the aws-region
* fix(resource manager): extend s3 to the storage of ds
fix the storageconfig init
* fix(resource manager): update e2e docker-compose
update e2e docker-compose
* fix(resource manager): extend s3 to the storage of ds
revent the e2e common.proprites
print the resource type in propertyUtil
* fix(resource manager): extend s3 to the storage of ds
1.println the properties
* fix(resource manager): println the s3 info
* fix(resource manager): extend s3 to the storage of ds
delete the info and upgrade the s3 info to e2e
* fix(resource manager): extend s3 to the storage of ds
add the bucket init
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): extend s3 to the storage of ds
upgrade the s3 endpoint
* fix(resource manager): withPathStyleAccessEnabled(true)
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): upgrade the s3client builder
* fix(resource manager): correct the s3 point to s3client
* fix(resource manager): update the constant BUCKET_NAME
* fix(resource manager): e2e s3 endpoint -> s3:9000
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* style(resource manager): add info to createBucket
* style(resource manager): debug the log
* ci(resource manager): test
test s3
* ci(ci): add INSERT INTO dolphinscheduler.t_ds_tenant (id, tenant_code, description, queue_id, create_time, update_time) VALUES(1, 'root', NULL, 1, NULL, NULL); to h2.sql
* fix(resource manager): update the h2 sql
* fix(resource manager): solve to delete the tenant
* style(resource manager): merge the style end delete the unuse s3 config
* fix(resource manager): extend s3 to the storage of ds
UPDATE the rename resources when s3
* fix(resource manager): extend s3 to the storage of ds
1.fix the code style of QuartzImpl
* fix(resource manager): extend s3 to the storage of ds
1.impoort restore_type to CommonUtils
* fix(resource manager): update the work thread
* fix(resource manager): update the baseTaskProcessor
* fix(resource manager): upgrade dolphinscheduler-standalone-server.xml
* fix(resource manager): add user Info to dolphinscheduler_h2.sql
* fix(resource manager): merge the resourceType to NONE
* style(upgrade the log level to info):
* fix(resource manager): sysnc the h2.sql
* fix(resource manager): update the merge the user tenant
* fix(resource manager): merge the resourcesServiceImpl
* fix(resource manager):
when the storage is s3 ,that the directory can't be renamed
* fix(resource manager): in s3 ,the directory cannot be renamed
* fix(resource manager): delete the deleteRenameDirectory in E2E
* fix(resource manager): check the style and recoverd the test
* fix(resource manager): delete the log.print(LoginUser)
2 years ago
|
|
|
List<String> list = storageOperate.vimFile(Mockito.any(), Mockito.anyString(), eq(1), eq(10));
|
|
|
|
Assertions.assertNotNull(list);
|
|
|
|
|
|
|
|
} catch (IOException e) {
|
|
|
|
logger.error("hadoop error", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Test
|
|
|
|
void testQueryBaseDir() {
|
|
|
|
User user = getUser();
|
|
|
|
when(userMapper.selectById(user.getId())).thenReturn(getUser());
|
|
|
|
when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
|
|
|
|
when(storageOperate.getDir(ResourceType.FILE, "123")).thenReturn("/dolphinscheduler/123/resources/");
|
|
|
|
try {
|
|
|
|
when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
|
|
|
|
Mockito.any())).thenReturn(getStorageEntityResource());
|
|
|
|
} catch (Exception e) {
|
|
|
|
logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest",
|
|
|
|
e);
|
|
|
|
}
|
|
|
|
Result<Object> result = resourcesService.queryResourceBaseDir(user, ResourceType.FILE);
|
|
|
|
assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
|
|
|
|
}
|
|
|
|
|
|
|
|
private Set<Integer> getSetIds() {
|
|
|
|
|
|
|
|
Set<Integer> resources = new HashSet<>();
|
|
|
|
resources.add(1);
|
|
|
|
return resources;
|
|
|
|
}
|
|
|
|
|
|
|
|
private Tenant getTenant() {
|
|
|
|
Tenant tenant = new Tenant();
|
|
|
|
tenant.setTenantCode("123");
|
|
|
|
return tenant;
|
|
|
|
}
|
|
|
|
|
|
|
|
private StorageEntity getStorageEntityResource() {
|
|
|
|
StorageEntity entity = new StorageEntity();
|
|
|
|
entity.setAlias("ResourcesServiceTest");
|
|
|
|
entity.setFileName("ResourcesServiceTest");
|
|
|
|
entity.setDirectory(false);
|
|
|
|
entity.setUserName("123");
|
|
|
|
entity.setType(ResourceType.FILE);
|
|
|
|
entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest");
|
|
|
|
|
|
|
|
return entity;
|
|
|
|
}
|
|
|
|
|
|
|
|
private StorageEntity getStorageEntityUdfResource() {
|
|
|
|
StorageEntity entity = new StorageEntity();
|
|
|
|
entity.setAlias("ResourcesServiceTest1.jar");
|
|
|
|
entity.setFileName("ResourcesServiceTest1.jar");
|
|
|
|
entity.setDirectory(false);
|
|
|
|
entity.setUserName("123");
|
|
|
|
entity.setType(ResourceType.UDF);
|
|
|
|
entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar");
|
|
|
|
|
|
|
|
return entity;
|
|
|
|
}
|
|
|
|
|
|
|
|
private UdfFunc getUdfFunc() {
|
|
|
|
|
|
|
|
UdfFunc udfFunc = new UdfFunc();
|
|
|
|
udfFunc.setId(1);
|
|
|
|
return udfFunc;
|
|
|
|
}
|
|
|
|
|
|
|
|
private UdfFunc getUdfFunc(int udfId) {
|
|
|
|
|
|
|
|
UdfFunc udfFunc = new UdfFunc();
|
|
|
|
udfFunc.setId(udfId);
|
|
|
|
return udfFunc;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<UdfFunc> getUdfFuncList() {
|
|
|
|
|
|
|
|
List<UdfFunc> udfFuncs = new ArrayList<>();
|
|
|
|
udfFuncs.add(getUdfFunc(1));
|
|
|
|
udfFuncs.add(getUdfFunc(2));
|
|
|
|
udfFuncs.add(getUdfFunc(3));
|
|
|
|
return udfFuncs;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<UdfFunc> getSingleUdfFuncList() {
|
|
|
|
return Collections.singletonList(getUdfFunc(3));
|
|
|
|
}
|
|
|
|
|
|
|
|
private User getUser() {
|
|
|
|
User user = new User();
|
|
|
|
user.setId(1);
|
|
|
|
user.setUserType(UserType.GENERAL_USER);
|
|
|
|
user.setTenantId(1);
|
[Feature-8612][RESOURCE] extend s3 to the storage of ds (#8637)
* feat(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.extend the type of storage
3.add the s3utils
to manager resource
4.automatic inject the storage in addition to your
config
* fix(resource manager): update the dependency
* fix(resource manager): extend s3 to the storage of ds
fix the constant of hadooputils
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager):
merge the unitTest:
1.TenantServiceImpl
2.ResourceServiceImpl
3.UserServiceImpl
* fix(resource manager): extend s3 to the storage of ds
merge the resourceServiceTest
* fix(resource manager): test cancel the test method
createTenant verifyTenant
* fix(resource manager): merge the code follow the check-result of sonar
* fix(resource manager): extend s3 to the storage of ds
fit the spell question
* fix(resource manager): extend s3 to the storage of ds
revert the common.properties
* fix(resource manager): extend s3 to the storage of ds
update the storageConfig with None
* fix(resource manager): extend s3 to the storage of ds
fix the judge of resourceType
* fix(resource manager): extend s3 to the storage of ds
undo the compile-mysql
* fix(resource manager): extend s3 to the storage of ds
delete hadoop aws
* fix(resource manager): extend s3 to the storage of ds
update the know-dependencies to delete aws 1.7.4
update the e2e
file-manager common.properties
* fix(resource manager): extend s3 to the storage of ds
update the aws-region
* fix(resource manager): extend s3 to the storage of ds
fix the storageconfig init
* fix(resource manager): update e2e docker-compose
update e2e docker-compose
* fix(resource manager): extend s3 to the storage of ds
revent the e2e common.proprites
print the resource type in propertyUtil
* fix(resource manager): extend s3 to the storage of ds
1.println the properties
* fix(resource manager): println the s3 info
* fix(resource manager): extend s3 to the storage of ds
delete the info and upgrade the s3 info to e2e
* fix(resource manager): extend s3 to the storage of ds
add the bucket init
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): extend s3 to the storage of ds
upgrade the s3 endpoint
* fix(resource manager): withPathStyleAccessEnabled(true)
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* fix(resource manager): upgrade the s3client builder
* fix(resource manager): correct the s3 point to s3client
* fix(resource manager): update the constant BUCKET_NAME
* fix(resource manager): e2e s3 endpoint -> s3:9000
* fix(resource manager): extend s3 to the storage of ds
1.fix some spell question
2.delete the import *
* style(resource manager): add info to createBucket
* style(resource manager): debug the log
* ci(resource manager): test
test s3
* ci(ci): add INSERT INTO dolphinscheduler.t_ds_tenant (id, tenant_code, description, queue_id, create_time, update_time) VALUES(1, 'root', NULL, 1, NULL, NULL); to h2.sql
* fix(resource manager): update the h2 sql
* fix(resource manager): solve to delete the tenant
* style(resource manager): merge the style end delete the unuse s3 config
* fix(resource manager): extend s3 to the storage of ds
UPDATE the rename resources when s3
* fix(resource manager): extend s3 to the storage of ds
1.fix the code style of QuartzImpl
* fix(resource manager): extend s3 to the storage of ds
1.impoort restore_type to CommonUtils
* fix(resource manager): update the work thread
* fix(resource manager): update the baseTaskProcessor
* fix(resource manager): upgrade dolphinscheduler-standalone-server.xml
* fix(resource manager): add user Info to dolphinscheduler_h2.sql
* fix(resource manager): merge the resourceType to NONE
* style(upgrade the log level to info):
* fix(resource manager): sysnc the h2.sql
* fix(resource manager): update the merge the user tenant
* fix(resource manager): merge the resourcesServiceImpl
* fix(resource manager):
when the storage is s3 ,that the directory can't be renamed
* fix(resource manager): in s3 ,the directory cannot be renamed
* fix(resource manager): delete the deleteRenameDirectory in E2E
* fix(resource manager): check the style and recoverd the test
* fix(resource manager): delete the log.print(LoginUser)
2 years ago
|
|
|
user.setTenantCode("tenantCode");
|
|
|
|
return user;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<String> getContent() {
|
|
|
|
List<String> contentList = new ArrayList<>();
|
|
|
|
contentList.add("test");
|
|
|
|
return contentList;
|
|
|
|
}
|
|
|
|
|
|
|
|
private List<Map<String, Object>> getResources() {
|
|
|
|
List<Map<String, Object>> resources = new ArrayList<>();
|
|
|
|
Map<String, Object> resource = new HashMap<>();
|
|
|
|
resource.put("id", 1);
|
|
|
|
resource.put("resource_ids", "1");
|
|
|
|
resources.add(resource);
|
|
|
|
return resources;
|
|
|
|
}
|
|
|
|
|
|
|
|
private static String getRandomStringWithLength(int length) {
|
|
|
|
Random r = new Random();
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
|
|
while (sb.length() < length) {
|
|
|
|
char c = (char) (r.nextInt(26) + 'a');
|
|
|
|
sb.append(c);
|
|
|
|
}
|
|
|
|
return sb.toString();
|
|
|
|
}
|
|
|
|
}
|